diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 0b99834aa88a..cd06b22e8a78 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -245,7 +245,7 @@ void GgmlOvDecoder::set_input_output() { if (src->op == GGML_OP_VIEW) { // Traverse upward through nested VIEW operations std::remove_reference_t view_chain; - auto current = src; + auto * current = src; while (current != nullptr) { auto current_name = get_tensor_ov_name(m_cgraph, current); @@ -612,9 +612,8 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (node->src[1]->view_src != nullptr) { if (node->src[3] != nullptr) { return 4; // decoder self-attention - } else { - return 5; // cross-attention or encoder self-attention - }; + } + return 5; // cross-attention or encoder self-attention } break; default: @@ -736,8 +735,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr bool rope_seen = false; for (int i = 0; i < cgraph->n_nodes; i++) { - auto * node = cgraph->nodes[i]; - std::string name = std::string(node->name); + ggml_tensor * node = cgraph->nodes[i]; const int attention_pattern_case = get_attention_pattern_case(node); if (attention_pattern_case != -1) { ggml_tensor * cache_k_permute = nullptr; @@ -948,7 +946,6 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, if (m_naive) { return input != nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)}; } - auto name = std::string(input->name); ov::PartialShape input_shape; if (is_inp_tok(input, op) || is_inp_pos(input, op)) { @@ -1474,7 +1471,7 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor void GgmlOvDecoder::dump_cgraph(const ggml_cgraph * cgraph, std::string & filename) { std::ofstream file(filename); if (!file.is_open()) { - std::cerr << "Failed to open file" << std::endl; + std::cerr << "Failed to open file" << '\n'; return; } @@ -1580,11 +1577,11 @@ void print_tensor_address_map(const ggml_cgraph * cgraph) { } } for (const auto & pair : address_map) { - std::cout << "Address: " << pair.first << std::endl; + std::cout << "Address: " << pair.first << '\n'; for (const auto & name : pair.second) { std::cout << name << " ; "; } - std::cout << std::endl << std::endl; + std::cout << "\n\n"; } } @@ -2226,7 +2223,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { std::cout << ", "; } } - std::cout << "]" << std::endl; + std::cout << "]" << '\n'; // print the src name & shape with the dynamic dim for debugging for (int j = 0; j < GGML_MAX_SRC; j++) { ggml_tensor * src = node->src[j]; @@ -2245,9 +2242,9 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { std::cout << ", "; } } - std::cout << "]" << std::endl; + std::cout << "]" << '\n'; } - std::cout << std::endl; + std::cout << '\n'; } } } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 7f9d45a48a87..056e39e87170 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -354,41 +354,41 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { void update_io(ggml_cgraph * cgraph); - inline static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_tok(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op == GGML_OP_NONE; } - inline static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_pos(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[1]; } // IMROPE packs 4 stacked position planes (t/h/w/e) into inp_pos, each of length // n_tokens; other modes carry a single position per token. - inline static int get_inp_pos_n_planes(const ggml_tensor * op) { + static int get_inp_pos_n_planes(const ggml_tensor * op) { return op->op_params[2] == GGML_ROPE_TYPE_IMROPE ? 4 : 1; } - inline static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_emb(const ggml_tensor * tensor, const ggml_tensor * op) { return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM; } - inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) || (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]); } - inline static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_mean(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_MUL_MAT && tensor == op->src[1] && tensor->op == GGML_OP_NONE && (tensor->flags & GGML_TENSOR_FLAG_INPUT) && tensor->type == GGML_TYPE_F32 && op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE; } - inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[2]; } // also returns true for cache_s and cache_r in SSM/DeltaNet models - inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) { if (tensor == nullptr) { return false; } @@ -396,14 +396,14 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { (op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor); } - inline static bool is_conv_state_writeback(const ggml_tensor * node) { + static bool is_conv_state_writeback(const ggml_tensor * node) { return node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) && node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr && node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src; } - inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor; } @@ -411,13 +411,13 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask; } - inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE && op->src[1]->op == GGML_OP_NONE; } // the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp) - inline static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) { + static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY; } @@ -481,5 +481,3 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { }; void print_tensor_address_map(const ggml_cgraph * cgraph); - -std::optional extract_layer_from_name(const std::string & name); diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 52e1a297c2d7..216e3b8a69e2 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -472,10 +472,6 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten switch (tensor->type) { case GGML_TYPE_MXFP4: - layout.is_u4 = true; - layout.is_symmetric = true; - break; - case GGML_TYPE_Q4_0: layout.is_u4 = true; layout.is_symmetric = true; diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 044b4da1c90f..02c5962238a7 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -28,12 +28,7 @@ #include #include -#ifndef _WIN32 -# include -# include -#endif - -#if defined(_WIN32) +#ifdef _WIN32 # define WIN32_LEAN_AND_MEAN # ifndef NOMINMAX # define NOMINMAX @@ -61,6 +56,7 @@ // - CPU repack buffer: tensor->extra stores tensor_traits with repacked data // ===================================================== +namespace { // Buffer context that manages per-tensor allocations (no contiguous buffer for weights) struct ggml_backend_openvino_buffer_context { int device; @@ -199,6 +195,7 @@ struct ggml_backend_openvino_buffer_type_context { int device; std::string name; }; +} // namespace // ===================================================== // Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS) @@ -258,14 +255,16 @@ void ggml_openvino_release_weight_buffers() { for (const auto & b : reg.buffers) { // Align down/up to page boundaries so madvise only drops whole pages // fully owned by this buffer. - const long page = sysconf(_SC_PAGESIZE); - uintptr_t start = reinterpret_cast(b.first); - uintptr_t end = start + b.second; - uintptr_t astart = (start + page - 1) & ~(uintptr_t) (page - 1); - uintptr_t aend = end & ~(uintptr_t) (page - 1); - if (aend > astart) { - if (madvise(reinterpret_cast(astart), aend - astart, MADV_DONTNEED) == 0) { - total += aend - astart; + const size_t page = (size_t) sysconf(_SC_PAGESIZE); + const uintptr_t ustart = reinterpret_cast(b.first); + const size_t offset_to_page = (page - (ustart & (page - 1))) & (page - 1); + if (b.second > offset_to_page) { + const size_t aligned_len = (b.second - offset_to_page) & ~(page - 1); + if (aligned_len > 0) { + char * astart = static_cast(b.first) + offset_to_page; + if (madvise(astart, aligned_len, MADV_DONTNEED) == 0) { + total += aligned_len; + } } } } @@ -876,11 +875,13 @@ GGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend) { return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_openvino_guid()); } +namespace { struct ggml_backend_openvino_device_context { int device; std::string name; std::string description; }; +} static const char * ggml_backend_openvino_device_get_name(ggml_backend_dev_t dev) { ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context; @@ -1588,9 +1589,11 @@ static const struct ggml_backend_device_i ggml_backend_openvino_device_interface /* .event_synchronize = */ NULL, }; +namespace { struct ggml_backend_openvino_reg_context { std::vector devices; }; +} static const char * ggml_backend_openvino_reg_get_name(ggml_backend_reg_t reg) { return GGML_OPENVINO_NAME; diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 93f9e8254aa6..824d24478290 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -34,6 +34,15 @@ #include #include +// From /src/common/transformations/include/transformations/utils/utils.hpp +namespace ov::op::util { +// From /src/common/transformations/include/transformations/utils/utils.hpp +bool get_single_value(const std::shared_ptr & const_node, + float & value, + bool check_value_range = true); +} // namespace ov::op::util + +namespace { void unpack_32_4(const uint8_t * data, uint8_t * dst) { std::fill_n(dst, 16, 0); for (int j = 0; j < 16; ++j) { @@ -48,11 +57,11 @@ void unpack_32_4(const uint8_t * data, uint8_t * dst) { } } -static constexpr size_t MXFP4_BLOCK_SIZE = 32; -static constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2; -static constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE; +constexpr size_t MXFP4_BLOCK_SIZE = 32; +constexpr size_t MXFP4_BLOCK_QS_SIZE = MXFP4_BLOCK_SIZE / 2; +constexpr size_t MXFP4_BLOCK_BYTES = sizeof(uint8_t) + MXFP4_BLOCK_QS_SIZE; -static void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) { +void pack_32_mxfp4_for_openvino(const uint8_t * data, uint8_t * dst) { for (int j = 0; j < static_cast(MXFP4_BLOCK_QS_SIZE); j += 2) { const uint8_t v0 = data[j] & 0x0F; const uint8_t v1 = (data[j + 1] & 0x0F) << 4; @@ -419,7 +428,7 @@ void extract_q6_k_data(const ggml_tensor * tensor, } } -static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { +inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t * d, uint8_t * m) { if (j < 4) { *d = q[j] & 63; *m = q[j + 4] & 63; @@ -514,9 +523,9 @@ void extract_q5_k_data(const ggml_tensor * tensor, ov::Output make_int8_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, - size_t group_size, - bool use_bias, - bool for_gather_matmul) { + size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, + bool use_bias = false, + bool for_gather_matmul = false) { ov::Shape orig_shape = weight.get_shape(); bool is_signed = (weight.get_element_type() == ov::element::i8); // Symmetric: signed weights, no ZP @@ -611,13 +620,24 @@ ov::Output make_int8_weights(ov::Tensor & weight, return std::make_shared(result, ov::element::f32); } +// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights +// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as +// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in +// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see +// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul +// directly from the resulting f16 dequant chain. +// +// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp +// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point); +// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not +// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern. // See make_int8_weights for the meaning of for_gather_matmul. ov::Output make_int4_weights(ov::Tensor & weight, ov::Tensor & scales, ov::Tensor & zp, - size_t group_size, - bool use_bias, - bool for_gather_matmul) { + size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, + bool use_bias = false, + bool for_gather_matmul = false) { ov::Shape orig_weight_shape = weight.get_shape(); bool is_signed = (weight.get_element_type() == ov::element::i4); // Symmetric: signed weights, no ZP @@ -746,13 +766,262 @@ ov::Output make_mxfp4_moe_packed_weights(ov::Tensor & weight) { return weights_node; } +void quantize_q4_0(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk) { + assert(k % qk == 0); + const int nb = k / qk; + + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4); // Signed i4 path + + if (!is_symmetric) { + auto * zp = static_cast(zp_arr.data()); + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + float max = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + if (amax < fabsf(v)) { + amax = fabsf(v); + max = v; + } + } + const float d = max / -8; + if (d == 0) { + scales[i] = ov::float16(1.0f); + if (i % 2 == 0) { + zp[i / 2] = 8; + } else { + zp[i / 2] |= (8 << 4); + } + memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2); + continue; + } + const float id = 1.0f / d; + scales[i] = ov::float16(d); + if (i % 2 == 0) { + zp[i / 2] = 8; + } else { + zp[i / 2] |= (8 << 4); + } + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f)); + const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f)); + weights[i * qk / 2 + j] = xi0 | (xi1 << 4); + } + } + } else { + // Symmetric: produce signed i4 values in [-8, 7] + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + float max = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + if (amax < fabsf(v)) { + amax = fabsf(v); + max = v; + } + } + const float d = max / -8; + if (d == 0) { + scales[i] = ov::float16(1.0f); + // i4 value 0 packed: 0x00 + memset(weights + i * qk / 2, 0, qk / 2); + continue; + } + const float id = 1.0f / d; + scales[i] = ov::float16(d); + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + // Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement. + int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0))); + int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1))); + weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4); + } + } + } +} + +// Asymmetric u4 quantization with a per-group scale and zero point. +// +// Unlike quantize_q4_0's unsigned branch, which pins the zero point to 8 and is therefore +// symmetric, this keeps a real per-group zero point, so a group whose values are not centred on +// zero does not waste half its range. +void quantize_q4_1_asym(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk) { + assert(k % qk == 0); + const int nb = k / qk; + + auto * weights = static_cast(weights_arr.data()); + auto * scales = scales_arr.data::value_type>(); + auto * zp = static_cast(zp_arr.data()); + + // u4 zero points are packed two per byte, low nibble first, indexed by group -- the same + // convention as the unsigned branch of quantize_q4_0. + auto store_zp = [zp](int i, uint8_t v) { + if (i % 2 == 0) { + zp[i / 2] = v & 0x0F; + } else { + zp[i / 2] |= (uint8_t) ((v & 0x0F) << 4); + } + }; + + for (int i = 0; i < nb; i++) { + float vmin = x[i * qk]; + float vmax = x[i * qk]; + for (int j = 1; j < qk; j++) { + const float v = x[i * qk + j]; + vmin = std::min(vmin, v); + vmax = std::max(vmax, v); + } + // Include 0 in the range so an all-positive or all-negative group still represents zero + // exactly -- these are weights, so an exact zero matters. + vmin = std::min(vmin, 0.0f); + vmax = std::max(vmax, 0.0f); + + const float d = (vmax - vmin) / 15.0f; + if (d == 0.0f) { + scales[i] = ov::float16(1.0f); + store_zp(i, 0); + memset(weights + i * qk / 2, 0, qk / 2); + continue; + } + const float id = 1.0f / d; + + // The zero point is itself a 4-bit integer, so round it and dequantize as (q - zq) * d. + const int zq = std::max(0, std::min(15, (int) lroundf(-vmin * id))); + scales[i] = ov::float16(d); + store_zp(i, (uint8_t) zq); + + for (int j = 0; j < qk / 2; ++j) { + const float x0 = x[i * qk + 2 * j] * id; + const float x1 = x[i * qk + 2 * j + 1] * id; + const uint8_t q0 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x0) + zq)); + const uint8_t q1 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x1) + zq)); + weights[i * qk / 2 + j] = (uint8_t) (q0 | (q1 << 4)); + } + } +} + +void quantize_q8_0(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk, + int64_t block_offset = 0) { + assert(k % qk == 0); + const int nb = k / qk; + + // block_offset lets a caller quantize a chunk of blocks into the right place in the + // output buffers (used for streaming requant). x points at this chunk's first block; + // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no + // nibble packing), so any block boundary is safe. + auto * weights = static_cast(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data::value_type>() + block_offset; + bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path + + if (!is_symmetric) { + auto * zp = static_cast(zp_arr.data()) + block_offset; + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + amax = std::max(amax, fabsf(v)); + } + const float d = amax / 127.0f; + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + zp[i] = 128; + for (int j = 0; j < qk; ++j) { + const float x0 = x[i * qk + j] * id; + const int8_t xi0 = roundf(x0); + weights[i * qk + j] = (uint8_t) (xi0 + 128); + } + } + } else { + // Symmetric: store signed int8 values directly + auto * signed_weights = reinterpret_cast(weights); + for (int i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + amax = std::max(amax, fabsf(v)); + } + const float d = amax / 127.0f; + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + for (int j = 0; j < qk; ++j) { + const float x0 = x[i * qk + j] * id; + signed_weights[i * qk + j] = (int8_t) roundf(x0); + } + } + } +} + +void quantize_q8_1(const float * x, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + int64_t k, + int64_t qk, + int64_t block_offset = 0) { + assert(k % qk == 0); + const int nb = k / qk; + + // See quantize_q8_0: block_offset places this chunk's output at the right block. + auto * weights = static_cast(weights_arr.data()) + block_offset * qk; + auto * scales = scales_arr.data::value_type>() + block_offset; + auto * zp = static_cast(zp_arr.data()) + block_offset; + for (int i = 0; i < nb; i++) { + float min = std::numeric_limits::max(); + float max = std::numeric_limits::lowest(); + + for (int j = 0; j < qk; j++) { + const float v = x[i * qk + j]; + min = std::min(v, min); + max = std::max(v, max); + } + + const float d = (max - min) / ((1 << 8) - 1); + const float id = d ? 1.0f / d : 0.0f; + scales[i] = ov::float16(d); + // zp = -min / scale (Q8_1 is asymmetric) + zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0; + + for (int j = 0; j < qk; ++j) { + const float x0 = (x[i * qk + j] - min) * id; + const uint8_t xi0 = roundf(x0); + weights[i * qk + j] = xi0; + } + } +} + // Extract quantized weights from tensor and create weight subgraph +// If weights/scales/zp are provided (non-empty), uses them as output buffers +// Otherwise allocates new ov::Tensors internally +// Returns the weight node (make_int4_weights or make_int8_weights result) std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, - const void * data, + const void * data, // Source data pointer (may differ from tensor->data) ov::Tensor & weights, ov::Tensor & scales, ov::Tensor & zp, - bool use_bias) { + // Use an exact f16 zero point (vs. a rounded integer one); always + // used for for_gather_matmul (3D MoE expert) weights regardless of + // this flag, and also settable explicitly for test-backend-ops. + bool use_bias = false) { // Create a temporary tensor for extraction functions that read from tensor->data ggml_tensor temp_tensor = *tensor; temp_tensor.data = const_cast(data); @@ -837,9 +1106,11 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, return result; } -// Requantize weights to target format, writing to provided buffers +// Requantize weights from tensor to target format, writing to provided buffers +// For F16 target, only weights buffer is used (scales/zp ignored) +// Returns the weight node std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, - const void * data, + const void * data, // Source data pointer ExtraQuantType requant_type, int64_t block_size, ov::Tensor & weights, @@ -933,6 +1204,7 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, result->set_friendly_name(tensor->name); return result; } +} // namespace OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, void * output_base_ptr, bool use_bias) { GGML_ASSERT(tensor != nullptr); @@ -1030,7 +1302,9 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo } else { result.weights = ov::Tensor(ov::element::f16, node_shape); } - ov::Tensor dummy_scales, dummy_zp; // Not used for F16 + // Not used for F16: + ov::Tensor dummy_scales; + ov::Tensor dummy_zp; result.weight_node = requantize_to_buffers(tensor, data, ExtraQuantType::F16, 0, result.weights, dummy_scales, dummy_zp); return result; @@ -1039,10 +1313,14 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo // Quantized path (normal extraction or quantized requant) // Create weight/scale/zp tensors - shared between both paths // For symmetric quantization, use signed types (i4/i8) and no ZP tensor - ov::element::Type weight_type = tensor->type == GGML_TYPE_MXFP4 ? - ov::element::f4e2m1 : - (layout.is_symmetric ? (layout.is_u4 ? ov::element::i4 : ov::element::i8) : - (layout.is_u4 ? ov::element::u4 : ov::element::u8)); + ov::element::Type weight_type; + if (tensor->type == GGML_TYPE_MXFP4) { + weight_type = ov::element::f4e2m1; + } else if (layout.is_symmetric) { + weight_type = layout.is_u4 ? ov::element::i4 : ov::element::i8; + } else { + weight_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; + } ov::Shape scale_shape = node_shape; scale_shape.back() /= layout.weights_per_block; @@ -1060,28 +1338,25 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo scale_shape.back() /= layout.weights_per_block; } + const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; + ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; + if (zp_is_f16) { + zp_type = ov::element::f16; + } + if (output_base_ptr) { uint8_t * buf_base = static_cast(output_base_ptr); result.weights = ov::Tensor(weight_type, node_shape, buf_base + layout.weights_offset); - const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; result.scales = ov::Tensor(scale_type, scale_shape, buf_base + layout.scales_offset); if (!layout.is_symmetric) { - ov::element::Type zp_type = - zp_is_f16 ? ov::element::f16 : (layout.is_u4 ? ov::element::u4 : ov::element::u8); result.zp = ov::Tensor(zp_type, scale_shape, buf_base + layout.zp_offset); } // else: result.zp remains default-constructed (empty) for symmetric } else { result.weights = ov::Tensor(weight_type, node_shape); - const ov::element::Type scale_type = tensor->type == GGML_TYPE_MXFP4 ? ov::element::f8e8m0 : ov::element::f16; result.scales = ov::Tensor(scale_type, scale_shape); if (!layout.is_symmetric) { - if (zp_is_f16) { - result.zp = ov::Tensor(ov::element::f16, scale_shape); - } else { - ov::element::Type zp_type = layout.is_u4 ? ov::element::u4 : ov::element::u8; - result.zp = ov::Tensor(zp_type, scale_shape); - } + result.zp = ov::Tensor(zp_type, scale_shape); } // else: result.zp remains default-constructed (empty) for symmetric } @@ -1096,246 +1371,3 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo return result; } - -void quantize_q4_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk) { - assert(k % qk == 0); - const int nb = k / qk; - - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); - bool is_symmetric = (weights_arr.get_element_type() == ov::element::i4); // Signed i4 path - - if (!is_symmetric) { - auto * zp = static_cast(zp_arr.data()); - for (int i = 0; i < nb; i++) { - float amax = 0.0f; - float max = 0.0f; - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - if (amax < fabsf(v)) { - amax = fabsf(v); - max = v; - } - } - const float d = max / -8; - if (d == 0) { - scales[i] = ov::float16(1.0f); - if (i % 2 == 0) { - zp[i / 2] = 8; - } else { - zp[i / 2] |= (8 << 4); - } - memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2); - continue; - } - const float id = 1.0f / d; - scales[i] = ov::float16(d); - if (i % 2 == 0) { - zp[i / 2] = 8; - } else { - zp[i / 2] |= (8 << 4); - } - for (int j = 0; j < qk / 2; ++j) { - const float x0 = x[i * qk + 2 * j] * id; - const float x1 = x[i * qk + 2 * j + 1] * id; - const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f)); - const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f)); - weights[i * qk / 2 + j] = xi0 | (xi1 << 4); - } - } - } else { - // Symmetric: produce signed i4 values in [-8, 7] - for (int i = 0; i < nb; i++) { - float amax = 0.0f; - float max = 0.0f; - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - if (amax < fabsf(v)) { - amax = fabsf(v); - max = v; - } - } - const float d = max / -8; - if (d == 0) { - scales[i] = ov::float16(1.0f); - // i4 value 0 packed: 0x00 - memset(weights + i * qk / 2, 0, qk / 2); - continue; - } - const float id = 1.0f / d; - scales[i] = ov::float16(d); - for (int j = 0; j < qk / 2; ++j) { - const float x0 = x[i * qk + 2 * j] * id; - const float x1 = x[i * qk + 2 * j + 1] * id; - // Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement. - int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0))); - int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1))); - weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4); - } - } - } -} - -// Asymmetric u4 quantization with a per-group scale and zero point. -// -// Unlike quantize_q4_0's unsigned branch, which pins the zero point to 8 and is therefore -// symmetric, this keeps a real per-group zero point, so a group whose values are not centred on -// zero does not waste half its range. -void quantize_q4_1_asym(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk) { - assert(k % qk == 0); - const int nb = k / qk; - - auto * weights = static_cast(weights_arr.data()); - auto * scales = scales_arr.data::value_type>(); - auto * zp = static_cast(zp_arr.data()); - - // u4 zero points are packed two per byte, low nibble first, indexed by group -- the same - // convention as the unsigned branch of quantize_q4_0. - auto store_zp = [zp](int i, uint8_t v) { - if (i % 2 == 0) { - zp[i / 2] = v & 0x0F; - } else { - zp[i / 2] |= (uint8_t) ((v & 0x0F) << 4); - } - }; - - for (int i = 0; i < nb; i++) { - float vmin = x[i * qk]; - float vmax = x[i * qk]; - for (int j = 1; j < qk; j++) { - const float v = x[i * qk + j]; - vmin = std::min(vmin, v); - vmax = std::max(vmax, v); - } - // Include 0 in the range so an all-positive or all-negative group still represents zero - // exactly -- these are weights, so an exact zero matters. - vmin = std::min(vmin, 0.0f); - vmax = std::max(vmax, 0.0f); - - const float d = (vmax - vmin) / 15.0f; - if (d == 0.0f) { - scales[i] = ov::float16(1.0f); - store_zp(i, 0); - memset(weights + i * qk / 2, 0, qk / 2); - continue; - } - const float id = 1.0f / d; - - // The zero point is itself a 4-bit integer, so round it and dequantize as (q - zq) * d. - const int zq = std::max(0, std::min(15, (int) lroundf(-vmin * id))); - scales[i] = ov::float16(d); - store_zp(i, (uint8_t) zq); - - for (int j = 0; j < qk / 2; ++j) { - const float x0 = x[i * qk + 2 * j] * id; - const float x1 = x[i * qk + 2 * j + 1] * id; - const uint8_t q0 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x0) + zq)); - const uint8_t q1 = (uint8_t) std::max(0, std::min(15, (int) lroundf(x1) + zq)); - weights[i * qk / 2 + j] = (uint8_t) (q0 | (q1 << 4)); - } - } -} - -void quantize_q8_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk, - int64_t block_offset) { - assert(k % qk == 0); - const int nb = k / qk; - - // block_offset lets a caller quantize a chunk of blocks into the right place in the - // output buffers (used for streaming requant). x points at this chunk's first block; - // outputs are advanced by block_offset blocks. Q8 has one scale/zp per block (no - // nibble packing), so any block boundary is safe. - auto * weights = static_cast(weights_arr.data()) + block_offset * qk; - auto * scales = scales_arr.data::value_type>() + block_offset; - bool is_symmetric = (weights_arr.get_element_type() == ov::element::i8); // Signed i8 path - - if (!is_symmetric) { - auto * zp = static_cast(zp_arr.data()) + block_offset; - for (int i = 0; i < nb; i++) { - float amax = 0.0f; - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - amax = std::max(amax, fabsf(v)); - } - const float d = amax / 127.0f; - const float id = d ? 1.0f / d : 0.0f; - scales[i] = ov::float16(d); - zp[i] = 128; - for (int j = 0; j < qk; ++j) { - const float x0 = x[i * qk + j] * id; - const int8_t xi0 = roundf(x0); - weights[i * qk + j] = (uint8_t) (xi0 + 128); - } - } - } else { - // Symmetric: store signed int8 values directly - auto * signed_weights = reinterpret_cast(weights); - for (int i = 0; i < nb; i++) { - float amax = 0.0f; - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - amax = std::max(amax, fabsf(v)); - } - const float d = amax / 127.0f; - const float id = d ? 1.0f / d : 0.0f; - scales[i] = ov::float16(d); - for (int j = 0; j < qk; ++j) { - const float x0 = x[i * qk + j] * id; - signed_weights[i * qk + j] = (int8_t) roundf(x0); - } - } - } -} - -void quantize_q8_1(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk, - int64_t block_offset) { - assert(k % qk == 0); - const int nb = k / qk; - - // See quantize_q8_0: block_offset places this chunk's output at the right block. - auto * weights = static_cast(weights_arr.data()) + block_offset * qk; - auto * scales = scales_arr.data::value_type>() + block_offset; - auto * zp = static_cast(zp_arr.data()) + block_offset; - for (int i = 0; i < nb; i++) { - float min = std::numeric_limits::max(); - float max = std::numeric_limits::lowest(); - - for (int j = 0; j < qk; j++) { - const float v = x[i * qk + j]; - min = std::min(v, min); - max = std::max(v, max); - } - - const float d = (max - min) / ((1 << 8) - 1); - const float id = d ? 1.0f / d : 0.0f; - scales[i] = ov::float16(d); - // zp = -min / scale (Q8_1 is asymmetric) - zp[i] = (d != 0.0f) ? (uint8_t) std::round(-min / d) : 0; - - for (int j = 0; j < qk; ++j) { - const float x0 = (x[i * qk + j] - min) * id; - const uint8_t xi0 = roundf(x0); - weights[i * qk + j] = xi0; - } - } -} diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index d5273727e87d..04fe0218a672 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -2,112 +2,12 @@ #include "ggml-openvino-extra.h" // For ExtraQuantType #include "ggml.h" -#include -#include #include +#include #include -void unpack_32_4(const uint8_t * data, uint8_t * dst); - -void extract_q4_0_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - -void extract_q4_1_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q5_1_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q8_0_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - -void unpack_256_4(const uint8_t * data, uint8_t * dst); - -void extract_q4_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q5_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - bool use_bias = false); - -void extract_q6_k_data(const ggml_tensor * tensor, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr); - -void extract_mxfp4_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr); - static constexpr size_t GGML_QUANTIZATION_GROUP_SIZE = 32; -// If for_gather_matmul is true, the weight tensor may be N-D (e.g. 3D MoE expert weights -// [n_expert, rows, cols]). The dequantization chain (Convert->[Subtract]->Multiply) is built as -// usual but left in f16 (no final Convert to f32) -- ov::pass::MarkDequantization (registered in -// translate_session.cpp) marks the chain so it survives model-build-time ConstantFolding -- see -// make_int8_weights.cpp/make_int4_weights.cpp. mul_mat_id.cpp constructs ov::op::internal::GatherMatmul -// directly from the resulting f16 dequant chain. -// -// When use_bias is true (explicitly, or implicitly because for_gather_matmul is true), the zp -// tensor is expected to hold an exact f16 bias value (rather than a rounded integer zero point); -// it is converted in place into an exact zero_point = -bias/scale and consumed via Subtract, not -// Add, so the chain still matches OpenVINO's Convert->Subtract->Multiply decompression pattern. -ov::Output make_int8_weights(ov::Tensor & weight, - ov::Tensor & scales, - ov::Tensor & zp, - size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, - bool use_bias = false, - bool for_gather_matmul = false); - -ov::Output make_int4_weights(ov::Tensor & weight, - ov::Tensor & scales, - ov::Tensor & zp, - size_t group_size = GGML_QUANTIZATION_GROUP_SIZE, - bool use_bias = false, - bool for_gather_matmul = false); - -ov::Output make_mxfp4_weights(ov::Tensor & weight, ov::Tensor & scales); - -ov::Output make_mxfp4_moe_packed_weights(ov::Tensor & weight); - -// Extract quantized weights from tensor and create weight subgraph -// If weights/scales/zp are provided (non-empty), uses them as output buffers -// Otherwise allocates new ov::Tensors internally -// Returns the weight node (make_int4_weights or make_int8_weights result) -std::shared_ptr extract_quantized_weights( - const ggml_tensor * tensor, - const void * data, // Source data pointer (may differ from tensor->data) - ov::Tensor & weights, - ov::Tensor & scales, - ov::Tensor & zp, - bool use_bias = false); // Use an exact f16 zero point (vs. a rounded integer one); always - // used for for_gather_matmul (3D MoE expert) weights regardless of - // this flag, and also settable explicitly for test-backend-ops. - -// Requantize weights from tensor to target format, writing to provided buffers -// For F16 target, only weights buffer is used (scales/zp ignored) -// Returns the weight node -std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, - const void * data, // Source data pointer - ExtraQuantType requant_type, - int64_t block_size, - ov::Tensor & weights, - ov::Tensor & scales, - ov::Tensor & zp); - inline const char * extra_quant_type_name(ExtraQuantType t) { switch (t) { case ExtraQuantType::F16: @@ -156,41 +56,3 @@ OvWeight process_weight_tensor( // always used for for_gather_matmul (3D MoE expert) weights // regardless of this flag, and also settable explicitly for // test-backend-ops. - -void quantize_q4_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk); -void quantize_q8_1(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk, - int64_t block_offset = 0); -void quantize_q4_1_asym(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk); -void quantize_q8_0(const float * x, - ov::Tensor & weights_arr, - ov::Tensor & scales_arr, - ov::Tensor & zp_arr, - int64_t k, - int64_t qk, - int64_t block_offset = 0); - -namespace ov { -namespace op { -namespace util { -// From /src/common/transformations/include/transformations/utils/utils.hpp -bool get_single_value(const std::shared_ptr & const_node, - float & value, - bool check_value_range = true); -} // namespace util -} // namespace op -} // namespace ov diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp index 3fc7028d88bc..3725fbd2252e 100644 --- a/ggml/src/ggml-openvino/model-cache.cpp +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -237,7 +237,8 @@ bool ggml_openvino_model_cache_verify_manifest(const std::string & path, if (!f.is_open()) { return false; } - std::string tag, val; + std::string tag; + std::string val; // header: fingerprint if (!(f >> tag >> val) || tag != "fingerprint" || val != hex64(fingerprint)) { return false; diff --git a/ggml/src/ggml-openvino/openvino/frontend.h b/ggml/src/ggml-openvino/openvino/frontend.h index 72134a3e8cf2..4e301d32e07a 100644 --- a/ggml/src/ggml-openvino/openvino/frontend.h +++ b/ggml/src/ggml-openvino/openvino/frontend.h @@ -12,7 +12,6 @@ namespace ggml { class FrontEnd { public: - using Ptr = std::shared_ptr; FrontEnd(); static std::shared_ptr convert(const InputModel::Ptr & model, bool naive = false); diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp index e54d700d421a..79bdbe87731e 100644 --- a/ggml/src/ggml-openvino/openvino/op/add_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/add_id.cpp @@ -20,7 +20,7 @@ namespace op { static ov::Output reshape_add_id_input_to_2d(const ov::Output & input, const ov::PartialShape & input_shape, const std::vector & dims) { - const auto actual_shape = input.get_partial_shape(); + const auto & actual_shape = input.get_partial_shape(); if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) { return input; } diff --git a/ggml/src/ggml-openvino/openvino/op/cont.cpp b/ggml/src/ggml-openvino/openvino/op/cont.cpp index 1d6cc6721260..9888f6b93fd3 100644 --- a/ggml/src/ggml-openvino/openvino/op/cont.cpp +++ b/ggml/src/ggml-openvino/openvino/op/cont.cpp @@ -3,12 +3,9 @@ #include "../op_table.h" #include "../utils.h" -#include -#include #include #include #include -#include namespace ov { namespace frontend { diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index 06547f3d2968..b06d01dcace0 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -195,7 +195,9 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output kv) { int64_t f = n_heads / n_heads_kv; if (f > 1 && n_heads_kv > 1) { - ov::Output kv_broadcast_shape, kv_unsqueezed, new_kv_shape; + ov::Output kv_broadcast_shape; + ov::Output kv_unsqueezed; + ov::Output new_kv_shape; auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2}); kv_unsqueezed = std::make_shared(kv, unsqueeze_axes); diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp index 07eeb3c8fd6d..8d07c90bfec1 100644 --- a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp @@ -196,7 +196,7 @@ static OutputVector translate_gated_delta_net_ref(const NodeContext & context) { } // Merge batch and head dims: [B*H_v, T, S_v] - auto merge_bh = [&](ov::Output x, int64_t last_dim) { + auto merge_bh = [&](const ov::Output & x, int64_t last_dim) { auto shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{B * H_v, T, last_dim}); return std::make_shared(x, shape, false); }; diff --git a/ggml/src/ggml-openvino/openvino/op/im2col.cpp b/ggml/src/ggml-openvino/openvino/op/im2col.cpp index 856e97f79d86..08b53f260d63 100644 --- a/ggml/src/ggml-openvino/openvino/op/im2col.cpp +++ b/ggml/src/ggml-openvino/openvino/op/im2col.cpp @@ -1,7 +1,6 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" -#include "ggml-impl.h" #include #include diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp index 0de6161bed85..a336924e14fa 100644 --- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -42,7 +42,7 @@ ov::Output slice_axis(const ov::Output & input, int64_t axis ov::Output static_shape_dims_or_shapeof(const ov::Output & input, const std::vector & dims) { - const auto partial_shape = input.get_partial_shape(); + const auto & partial_shape = input.get_partial_shape(); if (partial_shape.is_static()) { std::vector values; values.reserve(dims.size()); diff --git a/ggml/src/ggml-openvino/openvino/op/pad.cpp b/ggml/src/ggml-openvino/openvino/op/pad.cpp index d2b8611423cb..ae3d7be18eca 100644 --- a/ggml/src/ggml-openvino/openvino/op/pad.cpp +++ b/ggml/src/ggml-openvino/openvino/op/pad.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace ov { @@ -20,7 +21,7 @@ namespace { ov::Output translate_circular_pad(ov::Output input, const std::array & pads, const ov::Shape & input_shape) { - ov::Output result = input; + ov::Output result = std::move(input); const std::array pads_begin = {pads[6], pads[4], pads[2], pads[0]}; const std::array pads_end = {pads[7], pads[5], pads[3], pads[1]}; diff --git a/ggml/src/ggml-openvino/openvino/op/repeat.cpp b/ggml/src/ggml-openvino/openvino/op/repeat.cpp index d58b59e4e309..b7aeaa24fa82 100644 --- a/ggml/src/ggml-openvino/openvino/op/repeat.cpp +++ b/ggml/src/ggml-openvino/openvino/op/repeat.cpp @@ -1,7 +1,6 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" -#include "ggml.h" #include #include diff --git a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp index 9cbce7db0d50..25c9535454be 100644 --- a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp @@ -25,9 +25,7 @@ OutputVector translate_rms_norm(const NodeContext & context) { auto op_case = context.get_op_case(); ov::Output input_node; - if (op_case == 1) { - input_node = process_view_input_new(context, 0); - } else if (op_case == 2) { + if (op_case == 2) { auto ssm_state_size = context.get_ssm_state_size(); // The GDN op packs [attn | new_state] along the row axis; the state occupies the last // ssm_state_size * n_seqs rows. Slice it off (scaling by the active sequence count) to keep diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp index 56f5ceec9bb0..ca2d2dc08732 100644 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ b/ggml/src/ggml-openvino/openvino/op/view.cpp @@ -7,7 +7,6 @@ #include #include #include -#include namespace ov { namespace frontend { @@ -153,7 +152,8 @@ OutputVector translate_view(const NodeContext & context) { return {input}; } - int64_t src_elems = 1, dst_elems = 1; + int64_t src_elems = 1; + int64_t dst_elems = 1; for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) { if (src_shape[i].is_dynamic()) { return {input}; diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp index c9952b1d5201..04de2d008c72 100644 --- a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp +++ b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp @@ -84,7 +84,7 @@ bool KVStateSeqAxis::run_on_model(const std::shared_ptr & model) { // Readers still expect seq at dim 1. A reader that is itself the inverse // Transpose wanted seq at dim 2 all along, so drop it; give anything else the // inverse Transpose so its input is unchanged. - for (auto & reader : readers) { + for (const auto & reader : readers) { auto * node = reader.get_node(); if (ov::is_type(node)) { continue; diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index 3170c2e4ce05..e56a4e41d0b5 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -344,7 +344,7 @@ std::shared_ptr TranslateSession::translate_graph(const frontend::InputMo } }; - auto node_visitor = [&](std::shared_ptr decoder, int node_idx) { + auto node_visitor = [&](const std::shared_ptr & decoder, int node_idx) { auto converted_outputs = translate_node(decoder, node_idx); if (converted_outputs.empty()) { return; diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 8bb7678ee381..98a85e632a9e 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -1,7 +1,5 @@ #include "utils.h" -#include "ggml-impl.h" - #include #include #include @@ -28,13 +26,6 @@ namespace ov { namespace frontend { namespace ggml { -std::string getCurrentTime() { - std::time_t now = std::time(nullptr); - char buf[100]; - std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now)); - return buf; -} - void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs) { auto input_size = context.get_input_size(); FRONT_END_OP_CONVERSION_CHECK(input_size >= min_inputs, "Got less inputs than expected"); @@ -82,7 +73,7 @@ namespace { ov::Output rope_yarn_ramp_mix(int n_dims, const float corr_dims[2], float ext_factor) { int half_n_dims = n_dims / 2; std::vector dim_ids_vec(half_n_dims); - std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0); + std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0.0f); auto dim_ids = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, (size_t) half_n_dims}, dim_ids_vec); auto corr_low = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[0]}); auto corr_high = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[1]}); @@ -551,6 +542,7 @@ ov::Output process_view_input_new(const NodeContext & context, int inp if (tail_begin >= 0 && tail_end <= tail_src_elems) { std::vector flat_shape; + flat_shape.reserve(slice_dim); for (int i = 0; i < slice_dim; ++i) { flat_shape.push_back(static_cast(view_src_ggml_shape[i])); } diff --git a/ggml/src/ggml-openvino/openvino/utils.h b/ggml/src/ggml-openvino/openvino/utils.h index 5d4c3538664a..d9858f923655 100644 --- a/ggml/src/ggml-openvino/openvino/utils.h +++ b/ggml/src/ggml-openvino/openvino/utils.h @@ -14,8 +14,6 @@ namespace ggml { std::string getCurrentTime(); -void dump_ov_model(std::shared_ptr model); - void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs); int non_cont_dim(std::vector ne, std::vector nb); diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 44a9b2c78895..b1ee792fdb64 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -1,8 +1,8 @@ #include "utils.h" #include "ggml-impl.h" -#include "ggml-openvino.h" #include "ggml-openvino-extra.h" +#include "ggml-openvino.h" #include "ggml-openvino/ggml-decoder.h" #include "ggml.h" #include "model-cache.h" @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -39,42 +38,7 @@ #include #include -// Suppress deprecation warning for ov::Tensor::data() -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - -// Both execution paths use two cache levels: -// 1. Reuse this backend's decoder/request via graph_key and compatibility checks. -// 2. On a local miss, look up compiled_graph_key in the shared compilation cache, -// compile if needed, then create a private request from the compiled model. -// The shared lock covers compilation and frontend cleanup, never inference. -enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) { - ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; - try { - if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) { - std::string filename = "cgraph_ov.txt"; - GgmlOvDecoder::dump_cgraph(cgraph, filename); - } - - const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC"); - - GGML_ASSERT(ctx->runtime_context != nullptr); - std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context); - std::lock_guard execution_lock(r_ctx->execution_mutex); - - return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); - } catch (const ov::Exception & e) { - GGML_LOG_ERROR("GGML OpenVINO backend ov::Exception: %s\n", e.what()); - return GGML_STATUS_FAILED; - } catch (const std::exception & e) { - GGML_LOG_ERROR("GGML OpenVINO backend std::exception: %s\n", e.what()); - return GGML_STATUS_FAILED; - } catch (...) { - GGML_LOG_ERROR("GGML OpenVINO backend unknown exception\n"); - return GGML_STATUS_FAILED; - } -} - +namespace { // For a KV cache input, return an ov::Tensor sized to n_kv (== attention_size // for that layer) instead of the fully-allocated ctx_per_seq. Pre-conditions: // * non-static (CPU/GPU) backend, single sequence, seq_active_start == 0 @@ -85,9 +49,9 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) // n_kv rows no longer contain the live prefix // On any unmet pre-condition returns std::nullopt; the caller falls back to // the full-size tensor. -static std::optional try_make_kv_sliced_tensor(std::shared_ptr ggml_decoder, - const std::string & name, - const ggml_tensor * ggml_tensor) { +std::optional try_make_kv_sliced_tensor(const std::shared_ptr & ggml_decoder, + const std::string & name, + const ggml_tensor * ggml_tensor) { static const bool kv_slice_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE"); if (kv_slice_disabled) { return std::nullopt; @@ -125,7 +89,7 @@ static std::optional try_make_kv_sliced_tensor(std::shared_ptrget_shape(ggml_tensor); + ov::Shape full_shape = GgmlOvDecoder::get_shape(ggml_tensor); if (full_shape.size() != 4 || full_shape[0] != 1 || full_shape[1] != 1 || static_cast(full_shape[2]) != ctx_per_seq) { return std::nullopt; @@ -141,10 +105,10 @@ static std::optional try_make_kv_sliced_tensor(std::shared_ptrget_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); // } - return ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); + return ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); } -static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, bool stateful) { +uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, bool stateful) { const char * manual_gqa_env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN"); const bool manual_gqa_enabled = manual_gqa_env != nullptr ? ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0 : @@ -158,7 +122,7 @@ static uint64_t ggml_openvino_model_cache_extra_cfg(const std::string & device, return extra_cfg; } -static std::map> get_weight_names(ggml_cgraph * cgraph) { +std::map> get_weight_names(ggml_cgraph * cgraph) { std::map> names; for (const auto & name : GgmlOvDecoder::collect_weight_names(cgraph)) { names[name] = nullptr; @@ -170,8 +134,10 @@ static std::map> get_weight_names(ggml_cg // miss. Include topology, layouts, op parameters, constant extra inputs and weight // allocation identities. Never use a sampled weight hash or a graph name alone: // different models can have identical topology. OV buffer IDs survive address reuse. -static std::string compiled_graph_key(const ggml_cgraph * graph, const GgmlOvDecoder & decoder, - const std::string & device, int prefill_chunk_size = 0) { +std::string compiled_graph_key(const ggml_cgraph * graph, + const GgmlOvDecoder & decoder, + const std::string & device, + int prefill_chunk_size = 0) { std::string key; auto append = [&key](const auto & value) { key.append(reinterpret_cast(&value), sizeof(value)); @@ -243,8 +209,8 @@ static std::string compiled_graph_key(const ggml_cgraph * graph, const GgmlOvDec return has_weight_buffer_id ? key : std::string{}; } -ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, - std::shared_ptr infer_request, +ov::Tensor create_ov_output_tensor(const std::shared_ptr & ggml_decoder, + const std::shared_ptr & infer_request, int output_index, const ggml_tensor * ggml_tensor) { if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, std::string(ggml_tensor->name), ggml_tensor)) { @@ -260,7 +226,7 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, // } // } - auto output_type = ggml_decoder->get_ov_type(ggml_tensor); + auto output_type = GgmlOvDecoder::get_ov_type(ggml_tensor); ov::Shape output_shape; void * output_data = ggml_tensor->data; if (ggml_decoder->is_static()) { @@ -273,10 +239,10 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, // pointer instead so the OV tensor matches the model output exactly. if (ggml_tensor->op == GGML_OP_CPY && ggml_tensor->view_src != nullptr && ggml_nbytes(ggml_tensor) != ggml_nbytes(ggml_tensor->view_src)) { - output_shape = ggml_decoder->get_shape(ggml_tensor->view_src); + output_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src); output_data = ggml_tensor->view_src->data; } else { - output_shape = ggml_decoder->get_shape(ggml_tensor); + output_shape = GgmlOvDecoder::get_shape(ggml_tensor); } } ov::Tensor output_tensor(output_type, output_shape, output_data); @@ -286,7 +252,7 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, // Rewrite ggml's KV rows into a relayout state that keeps the sequence on dim 2. // ggml stores [seq][n_heads_kv * head_size]; the state wants [1, n_heads_kv, seq, head_size], // a different element order, so the rows are copied instead of reinterpreted. -static ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_heads_kv) { +ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_heads_kv) { const size_t rows = kv_tensor.get_shape()[2]; const size_t head_size = kv_tensor.get_shape()[3] / n_heads_kv; const size_t elem = kv_tensor.get_element_type().size(); @@ -303,528 +269,392 @@ static ov::Tensor kv_rows_to_seq_axis_2(const ov::Tensor & kv_tensor, size_t n_h return out; } -enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr r_ctx) { - auto & core = ov_singleton_core(); - const auto & config = ggml_openvino_get_compile_config(); - const auto & device = r_ctx->device; - const auto & stateful = r_ctx->stateful; - static auto is_static = false; +template void set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols, T zero_value = T{}) { + for (size_t i = 0; i < rows; ++i) { + size_t diag_col = std::min(i, cols - 1); + matrix[i * cols + diag_col] = zero_value; + } +} - static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); +ov::Tensor make_contiguous_split_input_tensor(const struct ggml_tensor * ggml_tensor, const ov::Shape & input_shape) { + const size_t element_size = ggml_type_size(ggml_tensor->type); + const size_t block_size = ggml_blck_size(ggml_tensor->type); - // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms - // on a Llama-1B decode graph. It is called once per graph_compute invocation but the - // graph shape is identical across all decode steps, so memoize by graph_key: compute - // graph_key first (a few hundred us), and if the same key is already in decoder_cache - // we know the graph is not splitted (only not-splitted graphs get inserted there). - graph_key key(cgraph); - bool key_seen = false; - if (!cache_disabled) { - std::lock_guard map_lock(r_ctx->ctx_mutex); - key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end(); - } + GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types"); - bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph); + const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor; + const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0; - if (is_naive(cgraph)) { - if (!model_is_splitted) { - return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); - } - } + std::vector source_data(ggml_nbytes(source_tensor)); + ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size()); - auto start_time = ggml_time_us(); + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + auto * dst = static_cast(input_tensor.data()); + size_t dst_offset = 0; - std::shared_ptr ggml_decoder; - std::shared_ptr infer_request; - ModelParams m_params; - ComputeParams c_params; - std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); + for (size_t i3 = 0; i3 < static_cast(ggml_tensor->ne[3]); ++i3) { + for (size_t i2 = 0; i2 < static_cast(ggml_tensor->ne[2]); ++i2) { + for (size_t i1 = 0; i1 < static_cast(ggml_tensor->ne[1]); ++i1) { + for (size_t i0 = 0; i0 < static_cast(ggml_tensor->ne[0]); ++i0) { + const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] + + i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0]; + std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size); + dst_offset += element_size; + } + } + } + } - const bool cache_enabled = !model_is_splitted && !cache_disabled; - bool cache_hit = false; + return input_tensor; +} - int64_t decoder_end_time; - int64_t conversion_end_time; - int64_t compile_end_time; - int64_t infer_end_time; - int64_t ov_raw_infer_start; +ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr & ggml_decoder, const std::string & name) { + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); - { - std::shared_ptr entry; - ModelParams old_m_params; + if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) { + return *sliced; + } - if (cache_enabled) { - std::lock_guard map_lock(r_ctx->ctx_mutex); - auto it = r_ctx->decoder_cache.find(key); - cache_hit = it != r_ctx->decoder_cache.end(); - if (cache_hit) { - entry = it->second; - } else { - r_ctx->clear_caches_locked(); - auto mutex = std::make_shared(); - entry = std::make_shared(mutex); - r_ctx->decoder_cache[key] = entry; - } - } else { - auto mutex = std::make_shared(); - entry = std::make_shared(mutex); - cache_hit = false; + if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { + auto * extra_base = static_cast(ggml_tensor->extra); + if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { + // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); + auto * tensor_extra = static_cast(extra_base); + return *tensor_extra->tensor; } + } - std::lock_guard lock(*(entry->mutex)); - cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0; + // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str()); + auto * input_data = ggml_tensor->data; + ov::Shape input_shape; + if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) { + // This case is added to make test-backend-ops work + input_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src); + } else { + input_shape = GgmlOvDecoder::get_shape(ggml_tensor); + } - if (cache_hit) { - ggml_decoder = entry->ptr; - old_m_params = ggml_decoder->get_model_params(); - if (!ggml_decoder->is_splited_model()) { - cache_hit = old_m_params.can_reuse_dynamically(m_params); - } - } + if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) { + return make_contiguous_split_input_tensor(ggml_tensor, input_shape); + } - std::vector ov_input_names; - std::vector ov_output_names; + auto input_tensor = ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape, input_data); + return input_tensor; +} - if (cache_hit) { - std::map> model_weights; - ggml_decoder->set_compute_params(c_params); - ggml_decoder->set_model_params(m_params); - if (old_m_params.kv_buffer_changed(m_params)) { - ggml_decoder->update_io(cgraph); - } - ggml_decoder->add_extra_inputs(); - { - std::lock_guard map_lock(r_ctx->ctx_mutex); - infer_request = r_ctx->infer_request_cache.at(key); - ov_input_names = r_ctx->ov_input_names_cache.at(key); - ov_output_names = r_ctx->ov_output_names_cache.at(key); - } +ov::Tensor get_ov_input_tensor(const std::shared_ptr & ggml_decoder, const std::string & param_name) { + ov::Tensor input_tensor; + auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name); + if (extra_input != ggml_decoder->get_model_extra_inputs().end()) { + input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape); + *input_tensor.data() = extra_input->second.value; + } else { + input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name); + } + return input_tensor; +} - if (stateful) { - const auto * inp_pos = get_inp_pos_tensor(cgraph); - int32_t * pos_data = (int32_t *) inp_pos->data; - auto pos_shape = ggml_decoder->get_shape(inp_pos); - if (pos_data[0] == 0) { - infer_request->reset_state(); - r_ctx->stateful_kv_size = pos_shape[3]; - } else if (r_ctx->stateful_kv_size == static_cast(pos_data[0])) { - r_ctx->stateful_kv_size += pos_shape[3]; - } else { - const size_t pos_begin = static_cast(pos_data[0]); - const bool refill = pos_begin > r_ctx->stateful_kv_size; +ov::Tensor get_ov_input_tensor_static_decode(const std::shared_ptr & ggml_decoder, + const std::string & param_name) { + // NPU decoding stage + if (ggml_decoder->get_model_extra_inputs().count(param_name)) { + return get_ov_input_tensor(ggml_decoder, param_name); + } + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); + const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); - // A refill seeds the state from ggml's KV cache, so it needs that cache to be a - // plain prefix: cell i must hold position i. An SWA layer keeps only the last - // n_swa positions, so once a position leaves the window ggml drops it and the - // remaining cells shift - cell i stops holding position i. While every position - // is still inside the window nothing has been dropped and the refill is sound. - if (refill && !ggml_decoder->get_model_params().swa_layers.empty()) { - const int n_swa = ggml_decoder->get_compute_params().swa_window; - if (n_swa < 0 || static_cast(n_swa) < pos_begin) { - GGML_LOG_ERROR( - "GGML OpenVINO backend stateful inference failed: cannot resume at position %zu from a " - "state that holds %zu tokens, because the sliding-window layers keep only the last %d " - "positions. Run without GGML_OPENVINO_STATEFUL_EXECUTION.\n", - pos_begin, r_ctx->stateful_kv_size, n_swa); - return GGML_STATUS_FAILED; - } - } + if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || + GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { + // IMROPE's inp_pos holds one value per t/h/w/e plane instead of a single position; + // with a single decode token the planes are still contiguous, so a flat copy works. + const int n_planes = GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ? GgmlOvDecoder::get_inp_pos_n_planes(op) : 1; + assert(ggml_tensor->ne[0] == n_planes); + ov::Shape input_shape = {1, 1, 1, (size_t) n_planes}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + std::memcpy(input_tensor.data(), ggml_tensor->data, n_planes * ggml_type_size(ggml_tensor->type)); + return input_tensor; + } - const bool relayout_enabled = - !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT"); + if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { + ov::Shape input_shape = {1, 1, 1, 1}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + int32_t inp_out_id = *((int32_t *) ggml_tensor->data); + assert(ggml_tensor->ne[0] == 1); + assert(inp_out_id == 0); + *input_tensor.data() = inp_out_id; + return input_tensor; + } - auto states = infer_request->query_state(); - for (auto state : states) { - auto state_tensor = state.get_state(); - auto state_tensor_shape = state_tensor.get_shape(); + if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { + size_t context_size = ggml_decoder->get_ctx_size(); + if (ggml_tensor->type == GGML_TYPE_F16) { + std::vector padded_data = + pad_input(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY)); + ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size}); + std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); + return input_tensor; + } - std::string state_name; - if (auto it = r_ctx->kv_state_input_name_map.find(state.get_name()); - it != r_ctx->kv_state_input_name_map.end()) { - state_name = it->second; - } + std::vector padded_data = pad_input(ggml_tensor, 1, context_size, -INFINITY); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size}); + auto * data_ptr = input_tensor.data(); + std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr); + return input_tensor; + } - // Which axis holds the sequence: pass::KVStateSeqAxis moves it from dim 1 - // to dim 2. The head count is still needed below, because only a 1-head - // state stays byte-compatible with ggml's cache buffer. gemma-4 12B mixes - // 1-head full layers with 8-head sliding layers, so it is per state. - int n_heads_kv = ggml_decoder->get_model_params().n_heads_kv; - if (auto layer = extract_layer_from_name(state_name); layer.has_value()) { - n_heads_kv = ggml_decoder->get_n_heads_kv_for_layer(layer.value()); - } - const bool relayout_this_state = relayout_enabled; - const size_t seq_axis = relayout_this_state ? 2 : 1; - const size_t head_axis = seq_axis == 2 ? 1 : 2; - - if (refill) { - if (state_name.empty()) { - GGML_LOG_ERROR( - "GGML OpenVINO backend stateful inference failed: no input found for the state\n"); - return GGML_STATUS_FAILED; - } - auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name); - if (relayout_this_state && n_heads_kv != 1) { - // several heads with seq on dim 2: not the same bytes as ggml's - // buffer, so the rows have to be copied into the new order - state_tensor = kv_rows_to_seq_axis_2(kv_tensor, (size_t) n_heads_kv); - } else { - ov::Shape refill_shape(4); - refill_shape[0] = state_tensor_shape[0]; - refill_shape[seq_axis] = kv_tensor.get_shape()[2]; - refill_shape[head_axis] = state_tensor_shape[head_axis]; - refill_shape[3] = state_tensor_shape[3]; - kv_tensor.set_shape(refill_shape); - state_tensor = kv_tensor; - } - state_tensor_shape = state_tensor.get_shape(); - } - // Only ever shrink to a prefix the source really has. Slicing past it used to - // surface as a bare ov::Exception from the ROI constructor. - if (state_tensor_shape[seq_axis] < pos_begin) { - GGML_LOG_ERROR( - "GGML OpenVINO backend stateful inference failed: state '%s' holds %zu tokens on axis " - "%zu, cannot resume at position %zu\n", - state.get_name().c_str(), state_tensor_shape[seq_axis], seq_axis, pos_begin); - return GGML_STATUS_FAILED; - } - ov::Coordinate begin = {0, 0, 0, 0}; - ov::Coordinate end(state_tensor_shape.begin(), state_tensor_shape.end()); - end[seq_axis] = pos_begin; - ov::Tensor new_state_tensor(state_tensor, begin, end); - state.set_state(new_state_tensor); - } - r_ctx->stateful_kv_size = pos_begin + pos_shape[3]; - } - } + return get_ov_input_tensor(ggml_decoder, param_name); +} - decoder_end_time = ggml_time_us(); - conversion_end_time = decoder_end_time; - compile_end_time = decoder_end_time; - } else { - // Compilation can mutate shared weight nodes, so serialize cold paths. - // The lock is released before binding tensors or running inference. - auto shared_cache = r_ctx->compiled_cache; - std::unique_lock compile_lock(shared_cache->mutex); - auto weight_names = get_weight_names(cgraph); - ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, - is_static, stateful, model_is_splitted); - const std::string shared_key = cache_enabled ? compiled_graph_key(cgraph, *ggml_decoder, device) : ""; - ov::CompiledModel shared_model; - bool imported = false; - auto shared_it = shared_cache->graphs.find(shared_key); - if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { - shared_model = shared_it->second.decode; - infer_request = std::make_shared(shared_model.create_infer_request()); - ov_input_names = shared_it->second.input_names; - ov_output_names = shared_it->second.output_names; - imported = true; - GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (dynamic)\n"); - } - // Fail fast: a cache-miss recompile feeds weight data to compile_model, but - // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU) - // may have already dropped the host weight pages - // (they would read as zeros). That mode requires stable graph shapes. - if (!imported && ggml_openvino_weight_buffers_released()) { - GGML_ABORT( - "ggml-openvino: a new graph needs to be compiled but host weight buffers were already " - "released via GGML_OPENVINO_RELEASE_WEIGHTS/GGML_OPENVINO_MEMORY_OPTIMIZE. This mode requires " - "stable graph shapes; disable host weight release for dynamic workloads."); - } - if (cache_enabled) { - std::lock_guard map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache.erase(key); - } +ov::Tensor get_ov_input_tensor_static_prefill(const std::shared_ptr & ggml_decoder, + const std::string & param_name, + int chunk_index) { + // NPU prompt processing stage + const size_t input_len = ggml_decoder->get_input_len(); + const size_t chunk_size = ggml_decoder->m_prefill_chunk_size; + const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size); + const size_t chunk_pad_size = chunk_size - chunk_valid_size; - // Frontend-level compiled-model cache (GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR): if this model - // was compiled before, import the saved blob and skip requant + convert + - // compile. Only the dynamic single-model path is cached (split models compile - // two graphs and are left to the plugin-level ov::cache_dir). The decoder is - // still needed for I/O mapping, but can be built without weight nodes since - // the weights are baked into the imported CompiledModel. - const std::string model_cache_dir = ggml_openvino_model_cache_dir(); - uint64_t model_fp = 0; - std::string blob_path, manifest_path; - // When the frontend model cache is active it supersedes the plugin-level - // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot - // be re-imported (import returns an uninitialized model). Strip cache_dir / - // cache_mode from the config used for the cached compile and the import. - ov::AnyMap mc_config = config; - if (!model_cache_dir.empty()) { - mc_config.erase("CACHE_DIR"); - mc_config.erase("CACHE_MODE"); - } - if (!imported && !model_cache_dir.empty() && !model_is_splitted) { - const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful); - model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, - 16, extra_cfg); - blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); - manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); + if (param_name == "chunk_valid_len") { + ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); + *input_tensor.data() = (int64_t) chunk_valid_size; + return input_tensor; + } + if (chunk_index > 0 && param_name == "cache_rs_reset_len") { + // The recurrent-state clear belongs to the start of the sequence. Re-applying it on every + // chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero + // length makes scale.cpp's keep-mask select every slot) after the first chunk. + ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); + *input_tensor.data() = 0; + return input_tensor; + } + if (ggml_decoder->get_model_extra_inputs().count(param_name)) { + return get_ov_input_tensor(ggml_decoder, param_name); + } + const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); + const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); - std::ifstream blob_in(blob_path, std::ios::binary); - bool blob_ok = blob_in.is_open(); - bool manifest_ok = blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp); - if (blob_ok && manifest_ok) { - int64_t import_start = ggml_time_us(); - try { - ov::CompiledModel cm; - auto remote_context = ggml_openvino_get_remote_context(); - if (remote_context.has_value()) { - cm = core.import_model(blob_in, remote_context.value(), mc_config); - } else { - cm = core.import_model(blob_in, device, mc_config); - } - // Lightweight decoder: names-only weight map (membership is all the - // decoder needs; weights live in the imported model). - std::map> weight_names; - for (const auto & n : GgmlOvDecoder::collect_weight_names(cgraph)) { - weight_names[n] = nullptr; - } - ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, - is_static, stateful, model_is_splitted); - infer_request = std::make_shared(cm.create_infer_request()); - shared_model = cm; - entry->ptr = ggml_decoder; - // Names must match the decoder's ggml-tensor keys. The non-cached - // path keys off Parameter/Result *friendly names* (set by the - // frontend); export_model preserves these, and each compiled-model - // port's node is exactly that Parameter/Result. Use the port nodes - // directly (NOT get_runtime_model(), whose graph differs and is - // unsafe to deref this way). - for (const auto & p : cm.inputs()) { - ov_input_names.push_back(p.get_node()->get_friendly_name()); - } - for (const auto & o : cm.outputs()) { - ov_output_names.push_back(o.get_node()->get_friendly_name()); - } - imported = true; - if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { - GGML_LOG_INFO(" - Model cache import time: %.3f ms \n", - (ggml_time_us() - import_start) / 1000.0); - } - GGML_LOG_INFO("ggml-openvino: model cache HIT %s\n", blob_path.c_str()); - } catch (const std::exception & e) { - GGML_LOG_WARN("ggml-openvino: model cache import failed (%s), recompiling\n", e.what()); - imported = false; - } + if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) { + // IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length + // input_len; pad every plane independently so they stay aligned to chunk_size. + const int n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op); + const size_t element_size = ggml_type_size(ggml_tensor->type); + ov::Shape input_shape = {1, 1, 1, (size_t) n_planes * chunk_size}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + for (int p = 0; p < n_planes; p++) { + const char * src = + (const char *) ggml_tensor->data + (p * input_len + chunk_index * chunk_size) * element_size; + char * dst = (char *) input_tensor.data() + p * chunk_size * element_size; + std::memcpy(dst, src, chunk_valid_size * element_size); + if (chunk_pad_size > 0) { + if (ggml_tensor->type == GGML_TYPE_I32) { + int32_t last_value = *((const int32_t *) src + chunk_valid_size - 1); + int32_t * out = (int32_t *) dst; + std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); + } else if (ggml_tensor->type == GGML_TYPE_I64) { + int64_t last_value = *((const int64_t *) src + chunk_valid_size - 1); + int64_t * out = (int64_t *) dst; + std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); + } else { + throw std::runtime_error("Unexpected tensor type for " + param_name); } } + } + return input_tensor; + } - std::shared_ptr model; - if (imported) { - decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || + GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { + ov::Shape input_shape = {1, 1, 1, chunk_size}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + // copy the chunk_index-th chunk from ggml_tensor + size_t element_size = ggml_type_size(ggml_tensor->type); + void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size; + std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size); + // pad the rest with last_value + 1, so that kv's of padded positions are inserted + // to the next row after the valids row in the kvcache + if (chunk_pad_size > 0) { + if (ggml_tensor->type == GGML_TYPE_I32) { + int32_t last_value = + *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); + int32_t * output_data = input_tensor.data(); + std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); + } else if (ggml_tensor->type == GGML_TYPE_I64) { + int64_t last_value = + *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); + int64_t * output_data = input_tensor.data(); + std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); } else { - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - - ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, - stateful, model_is_splitted); - decoder_end_time = ggml_time_us(); - - auto input_model = std::make_shared(ggml_decoder); - model = ov::frontend::ggml::FrontEnd::convert(input_model); - ggml_decoder->clear_model_weights(); - conversion_end_time = ggml_time_us(); - - if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { - char timestamped_filename[64]; - auto timestamp = (long long) ggml_time_us(); - snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp); - ov::serialize(model, timestamped_filename); - } - - // Use the cache-stripped config when the frontend model cache is active, so - // the resulting CompiledModel can be exported and later re-imported. - const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config; - ov::CompiledModel compiled_model; - auto remote_context = ggml_openvino_get_remote_context(); - if (remote_context.has_value()) { - compiled_model = core.compile_model(model, remote_context.value(), compile_config); - } else { - compiled_model = core.compile_model(model, device, compile_config); - } - compile_end_time = ggml_time_us(); - - // Export to the frontend model cache for next time. Publish the blob first, - // then the manifest, so a cache hit only sees fully written artifacts. - if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) { - try { - const std::string blob_tmp = blob_path + ".tmp"; - const std::string manifest_tmp = manifest_path + ".tmp"; - if (ggml_openvino_model_cache_write_manifest(manifest_tmp, cgraph, model_fp)) { - std::ofstream blob_out(blob_tmp, std::ios::binary | std::ios::trunc); - if (blob_out.is_open()) { - compiled_model.export_model(blob_out); - blob_out.close(); - if (blob_out.good()) { - if (std::rename(blob_tmp.c_str(), blob_path.c_str()) == 0 && - std::rename(manifest_tmp.c_str(), manifest_path.c_str()) == 0) { - GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str()); - } else { - std::remove(blob_tmp.c_str()); - std::remove(manifest_tmp.c_str()); - } - } else { - std::remove(blob_tmp.c_str()); - std::remove(manifest_tmp.c_str()); - } - } else { - std::remove(manifest_tmp.c_str()); - } - } - } catch (const std::exception & e) { - GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what()); - } - } - - infer_request = std::make_shared(compiled_model.create_infer_request()); - shared_model = compiled_model; - entry->ptr = ggml_decoder; - - for (const auto & ov_param : model->get_parameters()) { - ov_input_names.push_back(ov_param->get_friendly_name()); - } - for (const auto & ov_output : model->get_results()) { - ov_output_names.push_back(ov_output->get_friendly_name()); - } - } // end non-imported (compile) path - - entry->ptr = ggml_decoder; - if (!shared_key.empty() && shared_it == shared_cache->graphs.end()) { - shared_cache->graphs.emplace(shared_key, ov_compiled_graph{shared_model, {}, ov_input_names, - ov_output_names}); - } - if (cache_enabled) { - std::lock_guard map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache[key] = infer_request; - r_ctx->ov_input_names_cache[key] = ov_input_names; - r_ctx->ov_output_names_cache[key] = ov_output_names; + throw std::runtime_error("Unexpected tensor type for " + param_name); } + } + return input_tensor; + } - if (stateful && cache_enabled) { - const auto * inp_pos = get_inp_pos_tensor(cgraph); - auto pos_shape = ggml_decoder->get_shape(inp_pos); - // A freshly compiled model starts with an empty state, so it can only serve a - // sequence from its beginning. A non-zero start position means the KV history was - // built elsewhere (a restored ggml cache), which the state cannot adopt. - const int32_t pos_begin = ((int32_t *) inp_pos->data)[0]; - if (pos_begin != 0) { - GGML_LOG_ERROR( - "GGML OpenVINO backend stateful inference failed: a new model was compiled for a sequence that " - "starts at position %d, but its state is empty. Run without " - "GGML_OPENVINO_STATEFUL_EXECUTION.\n", - pos_begin); - return GGML_STATUS_FAILED; - } - r_ctx->stateful_kv_size = pos_shape[3]; - const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names(); - for (const auto & pair : kv_param_res_names) { - r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first; - } + if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { + size_t output_len = ggml_decoder->get_compute_params().output_len; + ov::Shape input_shape = {1, 1, 1, output_len}; + ov::Tensor input_tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), input_shape); + if (ggml_tensor->ne[0] == 0) { + *input_tensor.data() = 0; + } else { + auto * data_addr = input_tensor.data(); + for (size_t i = 0; i < output_len; i++) { + data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size; } } + return input_tensor; + } - for (size_t i = 0; i < ov_input_names.size(); i++) { - auto param_name = ov_input_names[i]; - auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name); - infer_request->set_input_tensor(i, input_tensor); - - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { - print_input_tensor_info(param_name, input_tensor); - } + if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) { + const size_t n_seqs = ggml_tensor->ne[1]; + const size_t src_stride = ggml_tensor->ne[0]; + const size_t copy_len = std::min(chunk_valid_size, src_stride - chunk_index * chunk_size); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, n_seqs, chunk_size}); + auto * dst = input_tensor.data(); + std::fill(dst, dst + n_seqs * chunk_size, 0.0f); + const auto * src = static_cast(ggml_tensor->data) + chunk_index * chunk_size; + for (size_t s = 0; s < n_seqs; s++) { + std::memcpy(dst + s * chunk_size, src + s * src_stride, copy_len * sizeof(float)); } + return input_tensor; + } - for (size_t i = 0; i < ov_output_names.size(); i++) { - // Debug-only outputs added via GGML_OPENVINO_DEBUG_NODE (see - // translate_session.cpp) have no corresponding ggml tensor; leave - // them unbound so OpenVINO allocates its own tensor for them, - // rather than aliasing a ggml buffer that may be overwritten by a - // later in-place op before we get to read it. - const auto & model_outputs = ggml_decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(ov_output_names[i]); - if (model_output_it == model_outputs.end()) { - continue; - } - auto * ggml_tensor = model_output_it->second; - if (ggml_nbytes(ggml_tensor) == 0) { - continue; - } - auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); + if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { + size_t cols = ggml_tensor->ne[0]; + size_t rows = ggml_tensor->ne[1]; + size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size); + size_t context_size = ggml_decoder->get_ctx_size(); + if (ggml_tensor->type == GGML_TYPE_F16) { + const auto * ggml_data = + static_cast(ggml_tensor->data) + chunk_index * chunk_size * cols; + std::vector padded_data = pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, + context_size, GGML_FP32_TO_FP16(-INFINITY)); + set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f)); + ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size}); + std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); + return input_tensor; } - ov_raw_infer_start = ggml_time_us(); - infer_request->infer(); - infer_end_time = ggml_time_us(); + const auto * ggml_data = static_cast(ggml_tensor->data) + chunk_index * chunk_size * cols; + std::vector padded_data = + pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY); + set_zero_diagonal(padded_data, chunk_size, context_size); + ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size}); + auto * data_ptr = input_tensor.data(); + std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr); + return input_tensor; + } - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || - ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - for (size_t i = 0; i < ov_output_names.size(); i++) { - const auto output_tensor = infer_request->get_output_tensor(i); - print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data()); - } - } + return get_ov_input_tensor(ggml_decoder, param_name); +} - if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { - GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); - GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); - if (!cache_hit) { - GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", - (conversion_end_time - decoder_end_time) / 1000.0); - GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); - } - GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); - GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0); - } +enum ggml_status naive_compute(ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config, + ov_compiled_model_cache & cache) { + if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { + return GGML_STATUS_SUCCESS; } - // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of - // every weight after compile, so the host weight buffers can be dropped to reclaim - // RSS. Release only while holding the compilation mutex so another context cannot - // be reading host weights during conversion/compilation. Pin the shared compiled - // models across backend teardown; a later context can create its own request without - // reading the dropped pages. A new, uncached graph still fails fast above. - if (cache_hit && ggml_openvino_release_weights_enabled(device)) { - std::lock_guard compile_lock(r_ctx->compiled_cache->mutex); - if (!ggml_openvino_weight_buffers_released()) { - ggml_openvino_release_weight_buffers(); - } + std::unique_lock compile_lock(cache.mutex); + bool naive = true; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); + auto decoder = std::make_shared(cgraph, model_weights); + auto input_model = std::make_shared(decoder); + auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + ov::serialize(model, "IR_naive.xml"); } - return GGML_STATUS_SUCCESS; -} + std::shared_ptr infer_request; + auto remote_context = ggml_openvino_get_remote_context(); + ov::AnyMap compile_config = config; + if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { + // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved + compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::PERFORMANCE; + } else { + compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::ACCURACY; + } + if (remote_context.has_value()) { + infer_request = std::make_shared( + core.compile_model(model, remote_context.value(), compile_config).create_infer_request()); + } else { + infer_request = std::make_shared( + core.compile_model(model, device, compile_config).create_infer_request()); + } + std::vector input_names; + std::vector output_names; + for (const auto & param : model->get_parameters()) { + input_names.push_back(param->get_friendly_name()); + } + for (const auto & result : model->get_results()) { + output_names.push_back(result->get_friendly_name()); + } + // Destroy the frontend graph under the compilation lock as well: it can + // still own edges into the shared weight nodes. + model.reset(); + input_model.reset(); + decoder->clear_model_weights(); + model_weights.clear(); + compile_lock.unlock(); -static ov::AnyMap without_npuw(const ov::AnyMap & config) { - ov::AnyMap out; - for (const auto & kv : config) { - if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") { + for (size_t i = 0; i < input_names.size(); i++) { + const auto & param_name = input_names[i]; + auto input_tensor = get_ov_input_tensor(decoder, param_name); + infer_request->set_input_tensor(i, input_tensor); + } + + // Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten + // when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE + + infer_request->infer(); + + for (size_t i = 0; i < output_names.size(); i++) { + auto output_tensor = infer_request->get_output_tensor(i); + const auto & model_outputs = decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(output_names[i]); + if (model_output_it == model_outputs.end()) { + // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into. + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + print_output_tensor_info(output_names[i], output_tensor, output_tensor.data()); + } continue; } - out.insert(kv); + auto * ggml_tensor = model_output_it->second; + std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size()); } - return out; + return GGML_STATUS_SUCCESS; } -enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr r_ctx) { +enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::shared_ptr & r_ctx) { auto & core = ov_singleton_core(); + const auto & config = ggml_openvino_get_compile_config(); + const auto & device = r_ctx->device; + const auto & stateful = r_ctx->stateful; + static auto is_static = false; - auto get_prefill_chunk_size = [] { - static const int chunk_size = []() { - int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE"); - return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256; - }(); - return chunk_size; - }; + static const bool cache_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); - // Normally NPU, but honors GGML_OPENVINO_DEVICE so GGML_OPENVINO_FORCE_STATIC can run the - // static-shape path on CPU/GPU to isolate translation bugs from NPUW/NPU-driver issues. - static std::string device = ggml_openvino_get_device_name(); - static auto is_static = true; - static auto stateful = false; + // is_model_splitted is O(n_nodes^2) plus a create_weight_nodes scan and takes ~20 ms + // on a Llama-1B decode graph. It is called once per graph_compute invocation but the + // graph shape is identical across all decode steps, so memoize by graph_key: compute + // graph_key first (a few hundred us), and if the same key is already in decoder_cache + // we know the graph is not splitted (only not-splitted graphs get inserted there). + graph_key key(cgraph); + bool key_seen = false; + if (!cache_disabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + key_seen = r_ctx->decoder_cache.find(key) != r_ctx->decoder_cache.end(); + } - auto prefill_chunk_size = get_prefill_chunk_size(); - const auto & config = ggml_openvino_get_compile_config(); + bool model_is_splitted = key_seen ? false : is_model_splitted(cgraph); if (is_naive(cgraph)) { - return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); + if (!model_is_splitted) { + return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); + } } auto start_time = ggml_time_us(); @@ -835,749 +665,913 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrne[0]; - } - graph_key key(cgraph); - static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + const bool cache_enabled = !model_is_splitted && !cache_disabled; bool cache_hit = false; - int64_t decoder_end_time; - int64_t conversion_end_time; - int64_t compile_end_time; - int64_t infer_end_time; - int64_t ov_raw_infer_start; - int64_t ov_raw_infer_total = 0; + int64_t decoder_end_time; + int64_t conversion_end_time; + int64_t compile_end_time; + int64_t infer_end_time; + int64_t ov_raw_infer_start; + + { + std::shared_ptr entry; + ModelParams old_m_params; + + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + auto it = r_ctx->decoder_cache.find(key); + cache_hit = it != r_ctx->decoder_cache.end(); + if (cache_hit) { + entry = it->second; + } else { + r_ctx->clear_caches_locked(); + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + r_ctx->decoder_cache[key] = entry; + } + } else { + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + cache_hit = false; + } + + std::lock_guard lock(*(entry->mutex)); + cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0; + + if (cache_hit) { + ggml_decoder = entry->ptr; + old_m_params = ggml_decoder->get_model_params(); + if (!ggml_decoder->is_splited_model()) { + cache_hit = old_m_params.can_reuse_dynamically(m_params); + } + } + + std::vector ov_input_names; + std::vector ov_output_names; + + if (cache_hit) { + std::map> model_weights; + ggml_decoder->set_compute_params(c_params); + ggml_decoder->set_model_params(m_params); + if (old_m_params.kv_buffer_changed(m_params)) { + ggml_decoder->update_io(cgraph); + } + ggml_decoder->add_extra_inputs(); + { + std::lock_guard map_lock(r_ctx->ctx_mutex); + infer_request = r_ctx->infer_request_cache.at(key); + ov_input_names = r_ctx->ov_input_names_cache.at(key); + ov_output_names = r_ctx->ov_output_names_cache.at(key); + } + + if (stateful) { + const auto * inp_pos = get_inp_pos_tensor(cgraph); + int32_t * pos_data = (int32_t *) inp_pos->data; + auto pos_shape = GgmlOvDecoder::get_shape(inp_pos); + if (pos_data[0] == 0) { + infer_request->reset_state(); + r_ctx->stateful_kv_size = pos_shape[3]; + } else if (r_ctx->stateful_kv_size == static_cast(pos_data[0])) { + r_ctx->stateful_kv_size += pos_shape[3]; + } else { + const size_t pos_begin = static_cast(pos_data[0]); + const bool refill = pos_begin > r_ctx->stateful_kv_size; + + // A refill seeds the state from ggml's KV cache, so it needs that cache to be a + // plain prefix: cell i must hold position i. An SWA layer keeps only the last + // n_swa positions, so once a position leaves the window ggml drops it and the + // remaining cells shift - cell i stops holding position i. While every position + // is still inside the window nothing has been dropped and the refill is sound. + if (refill && !ggml_decoder->get_model_params().swa_layers.empty()) { + const int n_swa = ggml_decoder->get_compute_params().swa_window; + if (n_swa < 0 || static_cast(n_swa) < pos_begin) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: cannot resume at position %zu from a " + "state that holds %zu tokens, because the sliding-window layers keep only the last %d " + "positions. Run without GGML_OPENVINO_STATEFUL_EXECUTION.\n", + pos_begin, r_ctx->stateful_kv_size, n_swa); + return GGML_STATUS_FAILED; + } + } + + const bool relayout_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT"); - std::shared_ptr entry; - ModelParams old_m_params; + auto states = infer_request->query_state(); + for (auto state : states) { + auto state_tensor = state.get_state(); + auto state_tensor_shape = state_tensor.get_shape(); - if (cache_enabled) { - std::lock_guard map_lock(r_ctx->ctx_mutex); - auto it = r_ctx->decoder_cache.find(key); - cache_hit = it != r_ctx->decoder_cache.end(); - if (cache_hit) { - entry = it->second; - } else { - r_ctx->clear_caches_locked(); - auto mutex = std::make_shared(); - entry = std::make_shared(mutex); - r_ctx->decoder_cache[key] = entry; - } - } else { - auto mutex = std::make_shared(); - entry = std::make_shared(mutex); - cache_hit = false; - } + std::string state_name; + if (auto it = r_ctx->kv_state_input_name_map.find(state.get_name()); + it != r_ctx->kv_state_input_name_map.end()) { + state_name = it->second; + } - std::lock_guard lock(*(entry->mutex)); - cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0 && - r_ctx->infer_request_cache_prefill.count(key) != 0; + // Which axis holds the sequence: pass::KVStateSeqAxis moves it from dim 1 + // to dim 2. The head count is still needed below, because only a 1-head + // state stays byte-compatible with ggml's cache buffer. gemma-4 12B mixes + // 1-head full layers with 8-head sliding layers, so it is per state. + int n_heads_kv = ggml_decoder->get_model_params().n_heads_kv; + if (auto layer = extract_layer_from_name(state_name); layer.has_value()) { + n_heads_kv = ggml_decoder->get_n_heads_kv_for_layer(layer.value()); + } + const bool relayout_this_state = relayout_enabled; + const size_t seq_axis = relayout_this_state ? 2 : 1; + const size_t head_axis = seq_axis == 2 ? 1 : 2; - if (cache_hit) { - ggml_decoder = entry->ptr; - old_m_params = ggml_decoder->get_model_params(); - cache_hit = old_m_params.can_reuse_statically(m_params); - } + if (refill) { + if (state_name.empty()) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: no input found for the state\n"); + return GGML_STATUS_FAILED; + } + auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name); + if (relayout_this_state && n_heads_kv != 1) { + // several heads with seq on dim 2: not the same bytes as ggml's + // buffer, so the rows have to be copied into the new order + state_tensor = kv_rows_to_seq_axis_2(kv_tensor, (size_t) n_heads_kv); + } else { + ov::Shape refill_shape(4); + refill_shape[0] = state_tensor_shape[0]; + refill_shape[seq_axis] = kv_tensor.get_shape()[2]; + refill_shape[head_axis] = state_tensor_shape[head_axis]; + refill_shape[3] = state_tensor_shape[3]; + kv_tensor.set_shape(refill_shape); + state_tensor = kv_tensor; + } + state_tensor_shape = state_tensor.get_shape(); + } + // Only ever shrink to a prefix the source really has. Slicing past it used to + // surface as a bare ov::Exception from the ROI constructor. + if (state_tensor_shape[seq_axis] < pos_begin) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: state '%s' holds %zu tokens on axis " + "%zu, cannot resume at position %zu\n", + state.get_name().c_str(), state_tensor_shape[seq_axis], seq_axis, pos_begin); + return GGML_STATUS_FAILED; + } + ov::Coordinate begin = {0, 0, 0, 0}; + ov::Coordinate end(state_tensor_shape.begin(), state_tensor_shape.end()); + end[seq_axis] = pos_begin; + ov::Tensor new_state_tensor(state_tensor, begin, end); + state.set_state(new_state_tensor); + } + r_ctx->stateful_kv_size = pos_begin + pos_shape[3]; + } + } - std::vector ov_input_names_local; - std::vector ov_output_names_local; + decoder_end_time = ggml_time_us(); + conversion_end_time = decoder_end_time; + compile_end_time = decoder_end_time; + } else { + // Compilation can mutate shared weight nodes, so serialize cold paths. + // The lock is released before binding tensors or running inference. + auto shared_cache = r_ctx->compiled_cache; + std::unique_lock compile_lock(shared_cache->mutex); + auto weight_names = get_weight_names(cgraph); + ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, is_static, + stateful, model_is_splitted); + const std::string shared_key = cache_enabled ? compiled_graph_key(cgraph, *ggml_decoder, device) : ""; + ov::CompiledModel shared_model; + bool imported = false; + auto shared_it = shared_cache->graphs.find(shared_key); + if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { + shared_model = shared_it->second.decode; + infer_request = std::make_shared(shared_model.create_infer_request()); + ov_input_names = shared_it->second.input_names; + ov_output_names = shared_it->second.output_names; + imported = true; + GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (dynamic)\n"); + } + // Fail fast: a cache-miss recompile feeds weight data to compile_model, but + // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU) + // may have already dropped the host weight pages + // (they would read as zeros). That mode requires stable graph shapes. + if (!imported && ggml_openvino_weight_buffers_released()) { + GGML_ABORT( + "ggml-openvino: a new graph needs to be compiled but host weight buffers were already " + "released via GGML_OPENVINO_RELEASE_WEIGHTS/GGML_OPENVINO_MEMORY_OPTIMIZE. This mode requires " + "stable graph shapes; disable host weight release for dynamic workloads."); + } + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache.erase(key); + } - if (cache_hit) { - std::map> model_weights; - ggml_decoder->m_is_prefill = is_prefill; - ggml_decoder->set_model_params(m_params); - ggml_decoder->set_compute_params(c_params); - if (old_m_params.kv_buffer_changed(m_params)) { - ggml_decoder->update_io(cgraph); - } - ggml_decoder->add_extra_inputs(); - { - std::lock_guard map_lock(r_ctx->ctx_mutex); - infer_request = - is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key); - ov_input_names_local = r_ctx->ov_input_names_cache.at(key); - ov_output_names_local = r_ctx->ov_output_names_cache.at(key); - } + // Frontend-level compiled-model cache (GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR): if this model + // was compiled before, import the saved blob and skip requant + convert + + // compile. Only the dynamic single-model path is cached (split models compile + // two graphs and are left to the plugin-level ov::cache_dir). The decoder is + // still needed for I/O mapping, but can be built without weight nodes since + // the weights are baked into the imported CompiledModel. + const std::string model_cache_dir = ggml_openvino_model_cache_dir(); + uint64_t model_fp = 0; + std::string blob_path; + std::string manifest_path; + // When the frontend model cache is active it supersedes the plugin-level + // ov::cache_dir: a blob exported from a model compiled WITH cache_dir cannot + // be re-imported (import returns an uninitialized model). Strip cache_dir / + // cache_mode from the config used for the cached compile and the import. + ov::AnyMap mc_config = config; + if (!model_cache_dir.empty()) { + mc_config.erase("CACHE_DIR"); + mc_config.erase("CACHE_MODE"); + } + if (!imported && !model_cache_dir.empty() && !model_is_splitted) { + const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful); + model_fp = + ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, 16, extra_cfg); + blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); + manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp); - decoder_end_time = ggml_time_us(); - conversion_end_time = decoder_end_time; - compile_end_time = decoder_end_time; - } else { - if (cache_enabled) { - std::lock_guard map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache.erase(key); - r_ctx->infer_request_cache_prefill.erase(key); - } + std::ifstream blob_in(blob_path, std::ios::binary); + bool blob_ok = blob_in.is_open(); + bool manifest_ok = + blob_ok && ggml_openvino_model_cache_verify_manifest(manifest_path, cgraph, model_fp); + if (blob_ok && manifest_ok) { + int64_t import_start = ggml_time_us(); + try { + ov::CompiledModel cm; + auto remote_context = ggml_openvino_get_remote_context(); + if (remote_context.has_value()) { + cm = core.import_model(blob_in, remote_context.value(), mc_config); + } else { + cm = core.import_model(blob_in, device, mc_config); + } + // Lightweight decoder: names-only weight map (membership is all the + // decoder needs; weights live in the imported model). + std::map> weight_names; + for (const auto & n : GgmlOvDecoder::collect_weight_names(cgraph)) { + weight_names[n] = nullptr; + } + ggml_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, + is_static, stateful, model_is_splitted); + infer_request = std::make_shared(cm.create_infer_request()); + shared_model = cm; + entry->ptr = ggml_decoder; + // Names must match the decoder's ggml-tensor keys. The non-cached + // path keys off Parameter/Result *friendly names* (set by the + // frontend); export_model preserves these, and each compiled-model + // port's node is exactly that Parameter/Result. Use the port nodes + // directly (NOT get_runtime_model(), whose graph differs and is + // unsafe to deref this way). + for (const auto & p : cm.inputs()) { + ov_input_names.push_back(p.get_node()->get_friendly_name()); + } + for (const auto & o : cm.outputs()) { + ov_output_names.push_back(o.get_node()->get_friendly_name()); + } + imported = true; + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { + GGML_LOG_INFO(" - Model cache import time: %.3f ms \n", + (ggml_time_us() - import_start) / 1000.0); + } + GGML_LOG_INFO("ggml-openvino: model cache HIT %s\n", blob_path.c_str()); + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache import failed (%s), recompiling\n", e.what()); + imported = false; + } + } + } - // Static execution shares a compiled prefill/decode pair. Each backend - // creates and retains its own requests for both phases. - auto shared_cache = r_ctx->compiled_cache; - std::unique_lock compile_lock(shared_cache->mutex); - auto weight_names = get_weight_names(cgraph); - auto local_decoder = std::make_shared( - cgraph, m_params, c_params, weight_names, is_static, stateful, false, is_prefill, prefill_chunk_size); - const std::string shared_key = cache_enabled ? - compiled_graph_key(cgraph, *local_decoder, device, prefill_chunk_size) : ""; - auto shared_it = shared_cache->graphs.find(shared_key); - if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { - auto & compiled = shared_it->second; - auto prefill_request = std::make_shared(compiled.prefill.create_infer_request()); - auto decode_request = no_kv_cache ? prefill_request : - std::make_shared(compiled.decode.create_infer_request()); - ggml_decoder = local_decoder; - entry->ptr = ggml_decoder; - infer_request = is_prefill ? prefill_request : decode_request; - ov_input_names_local = compiled.input_names; - ov_output_names_local = compiled.output_names; - r_ctx->infer_request_cache_prefill[key] = prefill_request; - r_ctx->infer_request_cache[key] = decode_request; - r_ctx->ov_input_names_cache[key] = ov_input_names_local; - r_ctx->ov_output_names_cache[key] = ov_output_names_local; - decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); - GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n"); - } else { std::shared_ptr model; - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - - auto ggml_decoder_prefill = std::make_shared( - cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size); - auto ggml_decoder_decode = - no_kv_cache ? ggml_decoder_prefill : - std::make_shared(cgraph, m_params, c_params, model_weights, is_static, - stateful, false, false, prefill_chunk_size); - decoder_end_time = ggml_time_us(); + if (imported) { + decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + } else { + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR"); - const auto dump_ir_timestamp = static_cast(ggml_time_us()); + ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, + stateful, model_is_splitted); + decoder_end_time = ggml_time_us(); - auto build_static_model = [&core, &compile_config, dump_ir, dump_ir_timestamp]( - std::shared_ptr decoder, - const char * tag, - std::shared_ptr & model, - ov::CompiledModel & compiled_model, - std::shared_ptr & infer_request, - int64_t & local_conversion_end_time, - int64_t & local_compile_end_time) { - auto input_model = std::make_shared(decoder); + auto input_model = std::make_shared(ggml_decoder); model = ov::frontend::ggml::FrontEnd::convert(input_model); - decoder->clear_model_weights(); - local_conversion_end_time = ggml_time_us(); + ggml_decoder->clear_model_weights(); + conversion_end_time = ggml_time_us(); - if (dump_ir) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { char timestamped_filename[64]; - snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag, - dump_ir_timestamp); + auto timestamp = (long long) ggml_time_us(); + snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp); ov::serialize(model, timestamped_filename); } - compiled_model = core.compile_model(model, device, compile_config); - infer_request = std::make_shared(compiled_model.create_infer_request()); - local_compile_end_time = ggml_time_us(); - }; - std::shared_ptr model_prefill; - std::shared_ptr model_decode; - ov::CompiledModel compiled_model_prefill; - ov::CompiledModel compiled_model_decode; - std::shared_ptr infer_request_prefill; - std::shared_ptr infer_request_decode; - int64_t prefill_conversion_end_time; - int64_t decode_conversion_end_time; - int64_t prefill_compile_end_time; - int64_t decode_compile_end_time; - build_static_model(ggml_decoder_prefill, "prefill", model_prefill, compiled_model_prefill, - infer_request_prefill, prefill_conversion_end_time, prefill_compile_end_time); - if (no_kv_cache) { - model_decode = model_prefill; - compiled_model_decode = compiled_model_prefill; - infer_request_decode = infer_request_prefill; - decode_conversion_end_time = prefill_conversion_end_time; - decode_compile_end_time = prefill_compile_end_time; - } else { - build_static_model(ggml_decoder_decode, "decode", model_decode, compiled_model_decode, infer_request_decode, - decode_conversion_end_time, decode_compile_end_time); - } - conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time); - compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time); - - model = is_prefill ? model_prefill : model_decode; - ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode; - infer_request = is_prefill ? infer_request_prefill : infer_request_decode; - entry->ptr = ggml_decoder; - - for (const auto & ov_param : model->get_parameters()) { - ov_input_names_local.push_back(ov_param->get_friendly_name()); - } - for (const auto & ov_output : model->get_results()) { - ov_output_names_local.push_back(ov_output->get_friendly_name()); - } - - if (!shared_key.empty()) { - shared_cache->graphs.emplace(shared_key, ov_compiled_graph{compiled_model_decode, compiled_model_prefill, - ov_input_names_local, ov_output_names_local}); - } - - if (cache_enabled) { - std::lock_guard map_lock(r_ctx->ctx_mutex); - r_ctx->infer_request_cache_prefill[key] = infer_request_prefill; - r_ctx->infer_request_cache[key] = infer_request_decode; - r_ctx->ov_input_names_cache[key] = ov_input_names_local; - r_ctx->ov_output_names_cache[key] = ov_output_names_local; - } - } - - } - - if (is_prefill) { - auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos); - for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) { - for (size_t i = 0; i < ov_input_names_local.size(); i++) { - auto param_name = ov_input_names_local[i]; - auto input_tensor = get_ov_input_tensor_static_prefill(ggml_decoder, param_name, chunk_index); - infer_request->set_input_tensor(i, input_tensor); - - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { - const auto input_tensor = infer_request->get_input_tensor(i); - print_input_tensor_info(param_name, input_tensor); + // Use the cache-stripped config when the frontend model cache is active, so + // the resulting CompiledModel can be exported and later re-imported. + const ov::AnyMap & compile_config = model_cache_dir.empty() ? config : mc_config; + ov::CompiledModel compiled_model; + auto remote_context = ggml_openvino_get_remote_context(); + if (remote_context.has_value()) { + compiled_model = core.compile_model(model, remote_context.value(), compile_config); + } else { + compiled_model = core.compile_model(model, device, compile_config); } - } + compile_end_time = ggml_time_us(); - for (size_t i = 0; i < ov_output_names_local.size(); i++) { - const auto & model_outputs = ggml_decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(ov_output_names_local[i]); - if (model_output_it == model_outputs.end()) { - continue; - } - auto * ggml_tensor = model_output_it->second; - if (ggml_nbytes(ggml_tensor) == 0) { - // Zero-row in-place writeback (e.g. the empty s_copy defrag remainder). The OV - // Result is the full cache, so binding it over this 0-byte buffer overflows it. - continue; + // Export to the frontend model cache for next time. Publish the blob first, + // then the manifest, so a cache hit only sees fully written artifacts. + if (!model_cache_dir.empty() && !model_is_splitted && model_fp != 0) { + try { + const std::string blob_tmp = blob_path + ".tmp"; + const std::string manifest_tmp = manifest_path + ".tmp"; + if (ggml_openvino_model_cache_write_manifest(manifest_tmp, cgraph, model_fp)) { + std::ofstream blob_out(blob_tmp, std::ios::binary | std::ios::trunc); + if (blob_out.is_open()) { + compiled_model.export_model(blob_out); + blob_out.close(); + if (blob_out.good()) { + if (std::rename(blob_tmp.c_str(), blob_path.c_str()) == 0 && + std::rename(manifest_tmp.c_str(), manifest_path.c_str()) == 0) { + GGML_LOG_INFO("ggml-openvino: model cache WROTE %s\n", blob_path.c_str()); + } else { + std::remove(blob_tmp.c_str()); + std::remove(manifest_tmp.c_str()); + } + } else { + std::remove(blob_tmp.c_str()); + std::remove(manifest_tmp.c_str()); + } + } else { + std::remove(manifest_tmp.c_str()); + } + } + } catch (const std::exception & e) { + GGML_LOG_WARN("ggml-openvino: model cache export failed: %s\n", e.what()); + } } - auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); - } - ov_raw_infer_start = ggml_time_us(); - infer_request->infer(); - ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start; + infer_request = std::make_shared(compiled_model.create_infer_request()); + shared_model = compiled_model; + entry->ptr = ggml_decoder; - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || - ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - for (size_t i = 0; i < ov_output_names_local.size(); i++) { - const auto output_tensor = infer_request->get_output_tensor(i); - print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); + for (const auto & ov_param : model->get_parameters()) { + ov_input_names.push_back(ov_param->get_friendly_name()); } - } - } - infer_end_time = ggml_time_us(); - } else { - for (size_t i = 0; i < ov_input_names_local.size(); i++) { - auto param_name = ov_input_names_local[i]; - auto input_tensor = get_ov_input_tensor_static_decode(ggml_decoder, param_name); - infer_request->set_input_tensor(i, input_tensor); - - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { - const auto input_tensor = infer_request->get_input_tensor(i); - print_input_tensor_info(param_name, input_tensor); - } - } + for (const auto & ov_output : model->get_results()) { + ov_output_names.push_back(ov_output->get_friendly_name()); + } + } // end non-imported (compile) path - for (size_t i = 0; i < ov_output_names_local.size(); i++) { - const auto & model_outputs = ggml_decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(ov_output_names_local[i]); - if (model_output_it == model_outputs.end()) { - continue; - } - auto * ggml_tensor = model_output_it->second; - if (ggml_nbytes(ggml_tensor) == 0) { - continue; + entry->ptr = ggml_decoder; + if (!shared_key.empty() && shared_it == shared_cache->graphs.end()) { + shared_cache->graphs.emplace(shared_key, + ov_compiled_graph{shared_model, {}, ov_input_names, ov_output_names}); } - auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); - } - - ov_raw_infer_start = ggml_time_us(); - infer_request->infer(); - infer_end_time = ggml_time_us(); - ov_raw_infer_total = infer_end_time - ov_raw_infer_start; - - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || - ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - for (size_t i = 0; i < ov_output_names_local.size(); i++) { - const auto output_tensor = infer_request->get_output_tensor(i); - print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache[key] = infer_request; + r_ctx->ov_input_names_cache[key] = ov_input_names; + r_ctx->ov_output_names_cache[key] = ov_output_names; } - } - } - - if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { - GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); - GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); - if (!cache_hit) { - GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", (conversion_end_time - decoder_end_time) / 1000.0); - GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); - } - GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); - GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", ov_raw_infer_total / 1000.0); - } - - return GGML_STATUS_SUCCESS; -} - -// Detect whether a cgraph is a split subgraph or not. -// Step 1 compares each node's recorded use_count with actual fan-out references in node->src. -// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split. -bool is_model_splitted(ggml_cgraph * cgraph) { - static const bool fallback_enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK") != 0; - if (!fallback_enabled) { - return false; - } - - // Backend op tests execute each node through ggml_graph_view(), which preserves the original - // graph use_counts while exposing only one node. Treat those single-node views as regular - // naive graphs so intermediate ops do not look like split-model fragments. - if (cgraph->n_nodes <= 1 && cgraph->n_leafs == 0) { - return false; - } - // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false. - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)]; - // TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future. - if ((cgraph->n_nodes <= 1 && use_count == 0) || - (cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr && - node->src[0]->op == GGML_OP_NONE)) { - return false; + if (stateful && cache_enabled) { + const auto * inp_pos = get_inp_pos_tensor(cgraph); + auto pos_shape = GgmlOvDecoder::get_shape(inp_pos); + // A freshly compiled model starts with an empty state, so it can only serve a + // sequence from its beginning. A non-zero start position means the KV history was + // built elsewhere (a restored ggml cache), which the state cannot adopt. + const int32_t pos_begin = ((int32_t *) inp_pos->data)[0]; + if (pos_begin != 0) { + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: a new model was compiled for a sequence that " + "starts at position %d, but its state is empty. Run without " + "GGML_OPENVINO_STATEFUL_EXECUTION.\n", + pos_begin); + return GGML_STATUS_FAILED; + } + r_ctx->stateful_kv_size = pos_shape[3]; + const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names(); + for (const auto & pair : kv_param_res_names) { + r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first; + } + } } - if (cgraph->n_nodes == 1 && - (cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) { - return false; + + for (size_t i = 0; i < ov_input_names.size(); i++) { + const auto & param_name = ov_input_names[i]; + auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name); + infer_request->set_input_tensor(i, input_tensor); + + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { + print_input_tensor_info(param_name, input_tensor); + } } - int input_use_count = 0; - for (int j = 0; j < cgraph->n_nodes; j++) { - ggml_tensor * other_node = cgraph->nodes[j]; - for (int k = 0; k < GGML_MAX_SRC; k++) { - if (other_node->src[k] == node) { - input_use_count++; - } + + for (size_t i = 0; i < ov_output_names.size(); i++) { + // Debug-only outputs added via GGML_OPENVINO_DEBUG_NODE (see + // translate_session.cpp) have no corresponding ggml tensor; leave + // them unbound so OpenVINO allocates its own tensor for them, + // rather than aliasing a ggml buffer that may be overwritten by a + // later in-place op before we get to read it. + const auto & model_outputs = ggml_decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(ov_output_names[i]); + if (model_output_it == model_outputs.end()) { + continue; + } + auto * ggml_tensor = model_output_it->second; + if (ggml_nbytes(ggml_tensor) == 0) { + continue; } + auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); + infer_request->set_output_tensor(i, output_tensor); } - if (use_count != input_use_count && node->op != GGML_OP_NONE) { - return true; + + ov_raw_infer_start = ggml_time_us(); + infer_request->infer(); + infer_end_time = ggml_time_us(); + + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + for (size_t i = 0; i < ov_output_names.size(); i++) { + const auto output_tensor = infer_request->get_output_tensor(i); + print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data()); + } } - } - // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check. - // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM - // use the name-only collector (no weight extraction); otherwise keep the original - // behavior of building (naive) weight nodes and take their names. - std::set model_weights; - if (ggml_openvino_reduce_compile_mem_enabled()) { - model_weights = GgmlOvDecoder::collect_weight_names(cgraph); - } else { - for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) { - model_weights.insert(kv.first); + + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { + GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); + GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); + if (!cache_hit) { + GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", + (conversion_end_time - decoder_end_time) / 1000.0); + GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); + } + GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); + GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0); } } - std::set model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes); - // leaf nodes - std::set model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs); - for (int i = 0; i < cgraph->n_nodes; i++) { - ggml_tensor * node = cgraph->nodes[i]; - for (int j = 0; j < GGML_MAX_SRC; j++) { - ggml_tensor * src = node->src[j]; - // the src is also not the model weights, we think the model is splitted. - // the src is also not in model leafs, we think the model is splitted. - if (src != nullptr && model_nodes.find(src) == model_nodes.end() && - model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false && - model_leafs.find(src) == model_leafs.end()) { - if (GgmlOvDecoder::is_inp_tok(src, node)) { - return false; - } - return true; - } + + // GGML_OPENVINO_RELEASE_WEIGHTS (or GGML_OPENVINO_MEMORY_OPTIMIZE on GPU): the plugin holds its own device copy of + // every weight after compile, so the host weight buffers can be dropped to reclaim + // RSS. Release only while holding the compilation mutex so another context cannot + // be reading host weights during conversion/compilation. Pin the shared compiled + // models across backend teardown; a later context can create its own request without + // reading the dropped pages. A new, uncached graph still fails fast above. + if (cache_hit && ggml_openvino_release_weights_enabled(device)) { + std::lock_guard compile_lock(r_ctx->compiled_cache->mutex); + if (!ggml_openvino_weight_buffers_released()) { + ggml_openvino_release_weight_buffers(); } } - return false; + + return GGML_STATUS_SUCCESS; } -bool is_naive(ggml_cgraph * cgraph) { - constexpr int naive_graph_size_threshold = 20; - int count = 0; - for (int i = 0; i < cgraph->n_nodes; i++) { - if (cgraph->nodes[i]->op != GGML_OP_NONE) { - count++; +ov::AnyMap without_npuw(const ov::AnyMap & config) { + ov::AnyMap out; + for (const auto & kv : config) { + if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") { + continue; } + out.insert(kv); } - return count < naive_graph_size_threshold; + return out; } -enum ggml_status naive_compute(ggml_cgraph * cgraph, - ov::Core & core, - const std::string & device, - const ov::AnyMap & config, - ov_compiled_model_cache & cache) { - if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { - return GGML_STATUS_SUCCESS; - } +enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared_ptr & r_ctx) { + auto & core = ov_singleton_core(); - std::unique_lock compile_lock(cache.mutex); - bool naive = true; - auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); - auto decoder = std::make_shared(cgraph, model_weights); - auto input_model = std::make_shared(decoder); - auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); - if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { - ov::serialize(model, "IR_naive.xml"); + auto get_prefill_chunk_size = [] { + static const int chunk_size = []() { + int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE"); + return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256; + }(); + return chunk_size; + }; + + // Normally NPU, but honors GGML_OPENVINO_DEVICE so GGML_OPENVINO_FORCE_STATIC can run the + // static-shape path on CPU/GPU to isolate translation bugs from NPUW/NPU-driver issues. + static std::string device = ggml_openvino_get_device_name(); + static auto is_static = true; + static auto stateful = false; + + auto prefill_chunk_size = get_prefill_chunk_size(); + const auto & config = ggml_openvino_get_compile_config(); + + if (is_naive(cgraph)) { + return naive_compute(cgraph, core, device, config, *r_ctx->compiled_cache); } + auto start_time = ggml_time_us(); + + std::shared_ptr ggml_decoder; std::shared_ptr infer_request; - auto remote_context = ggml_openvino_get_remote_context(); - ov::AnyMap compile_config = config; - if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { - // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved - compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::PERFORMANCE; - } else { - compile_config[ov::hint::execution_mode.name()] = ov::hint::ExecutionMode::ACCURACY; + ModelParams m_params; + ComputeParams c_params; + std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); + + const auto * inp_pos = get_inp_pos_tensor(cgraph); + const bool no_kv_cache = m_params.is_cacheless_attn; + const auto is_prefill = no_kv_cache ? true : get_is_prefill(cgraph, inp_pos); + const ov::AnyMap compile_config = no_kv_cache ? without_npuw(config) : config; + if (m_params.n_heads_kv == -1) { + prefill_chunk_size = inp_pos->ne[0]; } - if (remote_context.has_value()) { - infer_request = std::make_shared( - core.compile_model(model, remote_context.value(), compile_config).create_infer_request()); + graph_key key(cgraph); + static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + bool cache_hit = false; + + int64_t decoder_end_time; + int64_t conversion_end_time; + int64_t compile_end_time; + int64_t infer_end_time; + int64_t ov_raw_infer_start; + int64_t ov_raw_infer_total = 0; + + std::shared_ptr entry; + ModelParams old_m_params; + + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + auto it = r_ctx->decoder_cache.find(key); + cache_hit = it != r_ctx->decoder_cache.end(); + if (cache_hit) { + entry = it->second; + } else { + r_ctx->clear_caches_locked(); + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + r_ctx->decoder_cache[key] = entry; + } } else { - infer_request = - std::make_shared(core.compile_model(model, device, compile_config).create_infer_request()); - } - std::vector input_names; - std::vector output_names; - for (const auto & param : model->get_parameters()) { - input_names.push_back(param->get_friendly_name()); - } - for (const auto & result : model->get_results()) { - output_names.push_back(result->get_friendly_name()); + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + cache_hit = false; } - // Destroy the frontend graph under the compilation lock as well: it can - // still own edges into the shared weight nodes. - model.reset(); - input_model.reset(); - decoder->clear_model_weights(); - model_weights.clear(); - compile_lock.unlock(); - for (size_t i = 0; i < input_names.size(); i++) { - const auto & param_name = input_names[i]; - auto input_tensor = get_ov_input_tensor(decoder, param_name); - infer_request->set_input_tensor(i, input_tensor); - } + std::lock_guard lock(*(entry->mutex)); + cache_hit = cache_hit && entry->ptr && r_ctx->infer_request_cache.count(key) != 0 && + r_ctx->infer_request_cache_prefill.count(key) != 0; - // Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten - // when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE + if (cache_hit) { + ggml_decoder = entry->ptr; + old_m_params = ggml_decoder->get_model_params(); + cache_hit = old_m_params.can_reuse_statically(m_params); + } - infer_request->infer(); + std::vector ov_input_names_local; + std::vector ov_output_names_local; - for (size_t i = 0; i < output_names.size(); i++) { - auto output_tensor = infer_request->get_output_tensor(i); - const auto & model_outputs = decoder->get_model_outputs(); - auto model_output_it = model_outputs.find(output_names[i]); - if (model_output_it == model_outputs.end()) { - // Debug-only output added via GGML_OPENVINO_DEBUG_NODE; nothing to copy into. - if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || - ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - print_output_tensor_info(output_names[i], output_tensor, output_tensor.data()); - } - continue; + if (cache_hit) { + std::map> model_weights; + ggml_decoder->m_is_prefill = is_prefill; + ggml_decoder->set_model_params(m_params); + ggml_decoder->set_compute_params(c_params); + if (old_m_params.kv_buffer_changed(m_params)) { + ggml_decoder->update_io(cgraph); + } + ggml_decoder->add_extra_inputs(); + { + std::lock_guard map_lock(r_ctx->ctx_mutex); + infer_request = + is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key); + ov_input_names_local = r_ctx->ov_input_names_cache.at(key); + ov_output_names_local = r_ctx->ov_output_names_cache.at(key); } - auto * ggml_tensor = model_output_it->second; - std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size()); - } - return GGML_STATUS_SUCCESS; -} - -namespace { -template void set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols, T zero_value = T{}) { - for (size_t i = 0; i < rows; ++i) { - size_t diag_col = std::min(i, cols - 1); - matrix[i * cols + diag_col] = zero_value; - } -} -ov::Tensor make_contiguous_split_input_tensor(std::shared_ptr ggml_decoder, - const struct ggml_tensor * ggml_tensor, - const ov::Shape & input_shape) { - const size_t element_size = ggml_type_size(ggml_tensor->type); - const size_t block_size = ggml_blck_size(ggml_tensor->type); + decoder_end_time = ggml_time_us(); + conversion_end_time = decoder_end_time; + compile_end_time = decoder_end_time; + } else { + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache.erase(key); + r_ctx->infer_request_cache_prefill.erase(key); + } - GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types"); + // Static execution shares a compiled prefill/decode pair. Each backend + // creates and retains its own requests for both phases. + auto shared_cache = r_ctx->compiled_cache; + std::unique_lock compile_lock(shared_cache->mutex); + auto weight_names = get_weight_names(cgraph); + auto local_decoder = std::make_shared(cgraph, m_params, c_params, weight_names, is_static, + stateful, false, is_prefill, prefill_chunk_size); + const std::string shared_key = + cache_enabled ? compiled_graph_key(cgraph, *local_decoder, device, prefill_chunk_size) : ""; + auto shared_it = shared_cache->graphs.find(shared_key); + if (!shared_key.empty() && shared_it != shared_cache->graphs.end()) { + auto & compiled = shared_it->second; + auto prefill_request = std::make_shared(compiled.prefill.create_infer_request()); + auto decode_request = no_kv_cache ? + prefill_request : + std::make_shared(compiled.decode.create_infer_request()); + ggml_decoder = local_decoder; + entry->ptr = ggml_decoder; + infer_request = is_prefill ? prefill_request : decode_request; + ov_input_names_local = compiled.input_names; + ov_output_names_local = compiled.output_names; + r_ctx->infer_request_cache_prefill[key] = prefill_request; + r_ctx->infer_request_cache[key] = decode_request; + r_ctx->ov_input_names_cache[key] = ov_input_names_local; + r_ctx->ov_output_names_cache[key] = ov_output_names_local; + decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us(); + GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n"); + } else { + std::shared_ptr model; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor; - const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0; + auto ggml_decoder_prefill = std::make_shared( + cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size); + auto ggml_decoder_decode = + no_kv_cache ? ggml_decoder_prefill : + std::make_shared(cgraph, m_params, c_params, model_weights, is_static, + stateful, false, false, prefill_chunk_size); + decoder_end_time = ggml_time_us(); - std::vector source_data(ggml_nbytes(source_tensor)); - ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size()); + const bool dump_ir = ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR"); + const auto dump_ir_timestamp = static_cast(ggml_time_us()); - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - auto * dst = static_cast(input_tensor.data()); - size_t dst_offset = 0; + auto build_static_model = [&core, &compile_config, dump_ir, dump_ir_timestamp]( + const std::shared_ptr & decoder, const char * tag, + std::shared_ptr & model, ov::CompiledModel & compiled_model, + std::shared_ptr & infer_request, + int64_t & local_conversion_end_time, int64_t & local_compile_end_time) { + auto input_model = std::make_shared(decoder); + model = ov::frontend::ggml::FrontEnd::convert(input_model); + decoder->clear_model_weights(); + local_conversion_end_time = ggml_time_us(); - for (size_t i3 = 0; i3 < static_cast(ggml_tensor->ne[3]); ++i3) { - for (size_t i2 = 0; i2 < static_cast(ggml_tensor->ne[2]); ++i2) { - for (size_t i1 = 0; i1 < static_cast(ggml_tensor->ne[1]); ++i1) { - for (size_t i0 = 0; i0 < static_cast(ggml_tensor->ne[0]); ++i0) { - const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] + - i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0]; - std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size); - dst_offset += element_size; + if (dump_ir) { + char timestamped_filename[64]; + snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%s_%lld.xml", tag, + dump_ir_timestamp); + ov::serialize(model, timestamped_filename); } + + compiled_model = core.compile_model(model, device, compile_config); + infer_request = std::make_shared(compiled_model.create_infer_request()); + local_compile_end_time = ggml_time_us(); + }; + std::shared_ptr model_prefill; + std::shared_ptr model_decode; + ov::CompiledModel compiled_model_prefill; + ov::CompiledModel compiled_model_decode; + std::shared_ptr infer_request_prefill; + std::shared_ptr infer_request_decode; + int64_t prefill_conversion_end_time; + int64_t decode_conversion_end_time; + int64_t prefill_compile_end_time; + int64_t decode_compile_end_time; + build_static_model(ggml_decoder_prefill, "prefill", model_prefill, compiled_model_prefill, + infer_request_prefill, prefill_conversion_end_time, prefill_compile_end_time); + if (no_kv_cache) { + model_decode = model_prefill; + compiled_model_decode = compiled_model_prefill; + infer_request_decode = infer_request_prefill; + decode_conversion_end_time = prefill_conversion_end_time; + decode_compile_end_time = prefill_compile_end_time; + } else { + build_static_model(ggml_decoder_decode, "decode", model_decode, compiled_model_decode, + infer_request_decode, decode_conversion_end_time, decode_compile_end_time); } - } - } + conversion_end_time = std::max(prefill_conversion_end_time, decode_conversion_end_time); + compile_end_time = std::max(prefill_compile_end_time, decode_compile_end_time); - return input_tensor; -} + model = is_prefill ? model_prefill : model_decode; + ggml_decoder = is_prefill ? ggml_decoder_prefill : ggml_decoder_decode; + infer_request = is_prefill ? infer_request_prefill : infer_request_decode; + entry->ptr = ggml_decoder; -ov::Tensor convert_ggml_input_to_ov(std::shared_ptr ggml_decoder, const std::string & name) { - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); + for (const auto & ov_param : model->get_parameters()) { + ov_input_names_local.push_back(ov_param->get_friendly_name()); + } + for (const auto & ov_output : model->get_results()) { + ov_output_names_local.push_back(ov_output->get_friendly_name()); + } - if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) { - return *sliced; - } + if (!shared_key.empty()) { + shared_cache->graphs.emplace( + shared_key, ov_compiled_graph{compiled_model_decode, compiled_model_prefill, ov_input_names_local, + ov_output_names_local}); + } - if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { - auto * extra_base = static_cast(ggml_tensor->extra); - if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { - // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); - auto * tensor_extra = static_cast(extra_base); - return *tensor_extra->tensor; + if (cache_enabled) { + std::lock_guard map_lock(r_ctx->ctx_mutex); + r_ctx->infer_request_cache_prefill[key] = infer_request_prefill; + r_ctx->infer_request_cache[key] = infer_request_decode; + r_ctx->ov_input_names_cache[key] = ov_input_names_local; + r_ctx->ov_output_names_cache[key] = ov_output_names_local; + } } } - // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str()); - auto * input_data = ggml_tensor->data; - ov::Shape input_shape; - if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) { - // This case is added to make test-backend-ops work - input_shape = ggml_decoder->get_shape(ggml_tensor->view_src); - } else { - input_shape = ggml_decoder->get_shape(ggml_tensor); - } + if (is_prefill) { + auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos); + for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) { + for (size_t i = 0; i < ov_input_names_local.size(); i++) { + const auto & param_name = ov_input_names_local[i]; + auto input_tensor = get_ov_input_tensor_static_prefill(ggml_decoder, param_name, chunk_index); + infer_request->set_input_tensor(i, input_tensor); - if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) { - return make_contiguous_split_input_tensor(ggml_decoder, ggml_tensor, input_shape); - } + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { + const auto input_tensor = infer_request->get_input_tensor(i); + print_input_tensor_info(param_name, input_tensor); + } + } - auto input_tensor = ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape, input_data); - return input_tensor; -} -} // namespace + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto & model_outputs = ggml_decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(ov_output_names_local[i]); + if (model_output_it == model_outputs.end()) { + continue; + } + auto * ggml_tensor = model_output_it->second; + if (ggml_nbytes(ggml_tensor) == 0) { + // Zero-row in-place writeback (e.g. the empty s_copy defrag remainder). The OV + // Result is the full cache, so binding it over this 0-byte buffer overflows it. + continue; + } + auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); + infer_request->set_output_tensor(i, output_tensor); + } -ov::Tensor get_ov_input_tensor(std::shared_ptr ggml_decoder, const std::string & param_name) { - ov::Tensor input_tensor; - auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name); - if (extra_input != ggml_decoder->get_model_extra_inputs().end()) { - input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape); - *input_tensor.data() = extra_input->second.value; + ov_raw_infer_start = ggml_time_us(); + infer_request->infer(); + ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start; + + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto output_tensor = infer_request->get_output_tensor(i); + print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); + } + } + } + infer_end_time = ggml_time_us(); } else { - input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name); - } - return input_tensor; -} + for (size_t i = 0; i < ov_input_names_local.size(); i++) { + const auto & param_name = ov_input_names_local[i]; + auto input_tensor = get_ov_input_tensor_static_decode(ggml_decoder, param_name); + infer_request->set_input_tensor(i, input_tensor); -ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr ggml_decoder, - const std::string & param_name) { - // NPU decoding stage - if (ggml_decoder->get_model_extra_inputs().count(param_name)) { - return get_ov_input_tensor(ggml_decoder, param_name); - } - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); - const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { + const auto input_tensor = infer_request->get_input_tensor(i); + print_input_tensor_info(param_name, input_tensor); + } + } - if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || - GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { - // IMROPE's inp_pos holds one value per t/h/w/e plane instead of a single position; - // with a single decode token the planes are still contiguous, so a flat copy works. - const int n_planes = GgmlOvDecoder::is_inp_pos(ggml_tensor, op) ? GgmlOvDecoder::get_inp_pos_n_planes(op) : 1; - assert(ggml_tensor->ne[0] == n_planes); - ov::Shape input_shape = {1, 1, 1, (size_t) n_planes}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - std::memcpy(input_tensor.data(), ggml_tensor->data, n_planes * ggml_type_size(ggml_tensor->type)); - return input_tensor; - } + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto & model_outputs = ggml_decoder->get_model_outputs(); + auto model_output_it = model_outputs.find(ov_output_names_local[i]); + if (model_output_it == model_outputs.end()) { + continue; + } + auto * ggml_tensor = model_output_it->second; + if (ggml_nbytes(ggml_tensor) == 0) { + continue; + } + auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); + infer_request->set_output_tensor(i, output_tensor); + } - if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { - ov::Shape input_shape = {1, 1, 1, 1}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - int32_t inp_out_id = *((int32_t *) ggml_tensor->data); - assert(ggml_tensor->ne[0] == 1); - assert(inp_out_id == 0); - *input_tensor.data() = inp_out_id; - return input_tensor; - } + ov_raw_infer_start = ggml_time_us(); + infer_request->infer(); + infer_end_time = ggml_time_us(); + ov_raw_infer_total = infer_end_time - ov_raw_infer_start; - if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { - size_t context_size = ggml_decoder->get_ctx_size(); - if (ggml_tensor->type == GGML_TYPE_F16) { - std::vector padded_data = - pad_input(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY)); - ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size}); - std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); - return input_tensor; + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT") || + ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { + for (size_t i = 0; i < ov_output_names_local.size(); i++) { + const auto output_tensor = infer_request->get_output_tensor(i); + print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); + } } + } - std::vector padded_data = pad_input(ggml_tensor, 1, context_size, -INFINITY); - ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size}); - auto * data_ptr = input_tensor.data(); - std::copy(padded_data.begin(), padded_data.begin() + context_size, data_ptr); - return input_tensor; + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { + GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); + GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); + if (!cache_hit) { + GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", (conversion_end_time - decoder_end_time) / 1000.0); + GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); + } + GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); + GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", ov_raw_infer_total / 1000.0); } - return get_ov_input_tensor(ggml_decoder, param_name); + return GGML_STATUS_SUCCESS; } +} // namespace -ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr ggml_decoder, - const std::string & param_name, - int chunk_index) { - // NPU prompt processing stage - const size_t input_len = ggml_decoder->get_input_len(); - const size_t chunk_size = ggml_decoder->m_prefill_chunk_size; - const size_t chunk_valid_size = std::min(chunk_size, input_len - chunk_index * chunk_size); - const size_t chunk_pad_size = chunk_size - chunk_valid_size; +// Both execution paths use two cache levels: +// 1. Reuse this backend's decoder/request via graph_key and compatibility checks. +// 2. On a local miss, look up compiled_graph_key in the shared compilation cache, +// compile if needed, then create a private request from the compiled model. +// The shared lock covers compilation and frontend cleanup, never inference. +enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) { + ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; + try { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) { + std::string filename = "cgraph_ov.txt"; + GgmlOvDecoder::dump_cgraph(cgraph, filename); + } - if (param_name == "chunk_valid_len") { - ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); - *input_tensor.data() = (int64_t) chunk_valid_size; - return input_tensor; + const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC"); + + GGML_ASSERT(ctx->runtime_context != nullptr); + std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context); + std::lock_guard execution_lock(r_ctx->execution_mutex); + + return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); + } catch (const ov::Exception & e) { + GGML_LOG_ERROR("GGML OpenVINO backend ov::Exception: %s\n", e.what()); + return GGML_STATUS_FAILED; + } catch (const std::exception & e) { + GGML_LOG_ERROR("GGML OpenVINO backend std::exception: %s\n", e.what()); + return GGML_STATUS_FAILED; + } catch (...) { + GGML_LOG_ERROR("GGML OpenVINO backend unknown exception\n"); + return GGML_STATUS_FAILED; } - if (chunk_index > 0 && param_name == "cache_rs_reset_len") { - // The recurrent-state clear belongs to the start of the sequence. Re-applying it on every - // chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero - // length makes scale.cpp's keep-mask select every slot) after the first chunk. - ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); - *input_tensor.data() = 0; - return input_tensor; +} + +// Detect whether a cgraph is a split subgraph or not. +// Step 1 compares each node's recorded use_count with actual fan-out references in node->src. +// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split. +bool is_model_splitted(ggml_cgraph * cgraph) { + static const bool fallback_enabled = ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK") != 0; + if (!fallback_enabled) { + return false; } - if (ggml_decoder->get_model_extra_inputs().count(param_name)) { - return get_ov_input_tensor(ggml_decoder, param_name); + + // Backend op tests execute each node through ggml_graph_view(), which preserves the original + // graph use_counts while exposing only one node. Treat those single-node views as regular + // naive graphs so intermediate ops do not look like split-model fragments. + if (cgraph->n_nodes <= 1 && cgraph->n_leafs == 0) { + return false; } - const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); - const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); - if (GgmlOvDecoder::is_inp_pos(ggml_tensor, op) && GgmlOvDecoder::get_inp_pos_n_planes(op) > 1) { - // IMROPE: inp_pos stacks n_planes (t/h/w/e) position planes, each of length - // input_len; pad every plane independently so they stay aligned to chunk_size. - const int n_planes = GgmlOvDecoder::get_inp_pos_n_planes(op); - const size_t element_size = ggml_type_size(ggml_tensor->type); - ov::Shape input_shape = {1, 1, 1, (size_t) n_planes * chunk_size}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - for (int p = 0; p < n_planes; p++) { - const char * src = - (const char *) ggml_tensor->data + (p * input_len + chunk_index * chunk_size) * element_size; - char * dst = (char *) input_tensor.data() + p * chunk_size * element_size; - std::memcpy(dst, src, chunk_valid_size * element_size); - if (chunk_pad_size > 0) { - if (ggml_tensor->type == GGML_TYPE_I32) { - int32_t last_value = *((const int32_t *) src + chunk_valid_size - 1); - int32_t * out = (int32_t *) dst; - std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); - } else if (ggml_tensor->type == GGML_TYPE_I64) { - int64_t last_value = *((const int64_t *) src + chunk_valid_size - 1); - int64_t * out = (int64_t *) dst; - std::fill(out + chunk_valid_size, out + chunk_size, last_value + 1); - } else { - throw std::runtime_error("Unexpected tensor type for " + param_name); + // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false. + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)]; + // TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future. + if ((cgraph->n_nodes <= 1 && use_count == 0) || + (cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr && + node->src[0]->op == GGML_OP_NONE)) { + return false; + } + if (cgraph->n_nodes == 1 && + (cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) { + return false; + } + int input_use_count = 0; + for (int j = 0; j < cgraph->n_nodes; j++) { + ggml_tensor * other_node = cgraph->nodes[j]; + for (int k = 0; k < GGML_MAX_SRC; k++) { + if (other_node->src[k] == node) { + input_use_count++; } } } - return input_tensor; - } - - if (GgmlOvDecoder::is_inp_tok(ggml_tensor, op) || GgmlOvDecoder::is_inp_pos(ggml_tensor, op) || - GgmlOvDecoder::is_kv_idx(ggml_tensor, op)) { - ov::Shape input_shape = {1, 1, 1, chunk_size}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - // copy the chunk_index-th chunk from ggml_tensor - size_t element_size = ggml_type_size(ggml_tensor->type); - void * input_data = (char *) ggml_tensor->data + chunk_index * chunk_size * element_size; - std::memcpy(input_tensor.data(), input_data, chunk_valid_size * element_size); - // pad the rest with last_value + 1, so that kv's of padded positions are inserted - // to the next row after the valids row in the kvcache - if (chunk_pad_size > 0) { - if (ggml_tensor->type == GGML_TYPE_I32) { - int32_t last_value = - *((int32_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); - int32_t * output_data = input_tensor.data(); - std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); - } else if (ggml_tensor->type == GGML_TYPE_I64) { - int64_t last_value = - *((int64_t *) ggml_tensor->data + (chunk_index * chunk_size + chunk_valid_size - 1)); - int64_t * output_data = input_tensor.data(); - std::fill(output_data + chunk_valid_size, output_data + chunk_size, last_value + 1); - } else { - throw std::runtime_error("Unexpected tensor type for " + param_name); - } + if (use_count != input_use_count && node->op != GGML_OP_NONE) { + return true; } - return input_tensor; } - - if (GgmlOvDecoder::is_output_idx(ggml_tensor, op)) { - size_t output_len = ggml_decoder->get_compute_params().output_len; - ov::Shape input_shape = {1, 1, 1, output_len}; - ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); - if (ggml_tensor->ne[0] == 0) { - *input_tensor.data() = 0; - } else { - auto * data_addr = input_tensor.data(); - for (size_t i = 0; i < output_len; i++) { - data_addr[i] = ((int32_t *) ggml_tensor->data)[i] % chunk_size; - } + // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check. + // Only weight-name membership is needed below. With GGML_OPENVINO_REDUCE_COMPILE_MEM + // use the name-only collector (no weight extraction); otherwise keep the original + // behavior of building (naive) weight nodes and take their names. + std::set model_weights; + if (ggml_openvino_reduce_compile_mem_enabled()) { + model_weights = GgmlOvDecoder::collect_weight_names(cgraph); + } else { + for (const auto & kv : GgmlOvDecoder::create_weight_nodes(cgraph, true)) { + model_weights.insert(kv.first); } - return input_tensor; } - - if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) { - const size_t n_seqs = ggml_tensor->ne[1]; - const size_t src_stride = ggml_tensor->ne[0]; - const size_t copy_len = std::min(chunk_valid_size, src_stride - chunk_index * chunk_size); - ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, n_seqs, chunk_size}); - auto * dst = input_tensor.data(); - std::fill(dst, dst + n_seqs * chunk_size, 0.0f); - const auto * src = static_cast(ggml_tensor->data) + chunk_index * chunk_size; - for (size_t s = 0; s < n_seqs; s++) { - std::memcpy(dst + s * chunk_size, src + s * src_stride, copy_len * sizeof(float)); + std::set model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes); + // leaf nodes + std::set model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs); + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + ggml_tensor * src = node->src[j]; + // the src is also not the model weights, we think the model is splitted. + // the src is also not in model leafs, we think the model is splitted. + if (src != nullptr && model_nodes.find(src) == model_nodes.end() && + model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false && + model_leafs.find(src) == model_leafs.end()) { + if (GgmlOvDecoder::is_inp_tok(src, node)) { + return false; + } + return true; + } } - return input_tensor; } + return false; +} - if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { - size_t cols = ggml_tensor->ne[0]; - size_t rows = ggml_tensor->ne[1]; - size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size); - size_t context_size = ggml_decoder->get_ctx_size(); - if (ggml_tensor->type == GGML_TYPE_F16) { - const auto * ggml_data = - static_cast(ggml_tensor->data) + chunk_index * chunk_size * cols; - std::vector padded_data = pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, - context_size, GGML_FP32_TO_FP16(-INFINITY)); - set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f)); - ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size}); - std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); - return input_tensor; +bool is_naive(ggml_cgraph * cgraph) { + constexpr int naive_graph_size_threshold = 20; + int count = 0; + for (int i = 0; i < cgraph->n_nodes; i++) { + if (cgraph->nodes[i]->op != GGML_OP_NONE) { + count++; } - - const auto * ggml_data = static_cast(ggml_tensor->data) + chunk_index * chunk_size * cols; - std::vector padded_data = - pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY); - set_zero_diagonal(padded_data, chunk_size, context_size); - ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, chunk_size, context_size}); - auto * data_ptr = input_tensor.data(); - std::copy(padded_data.begin(), padded_data.begin() + chunk_size * context_size, data_ptr); - return input_tensor; } - - return get_ov_input_tensor(ggml_decoder, param_name); + return count < naive_graph_size_threshold; } size_t checksum(const void * data, size_t size) { @@ -1651,15 +1645,15 @@ bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor) { std::cout << "Input name: " << name << ", Input shape: " << tensor.get_shape() << ", Address: " << tensor.data() - << std::endl; + << '\n'; switch (tensor.get_element_type()) { case ov::element::f32: { if (name.find("self_kq_mask") == std::string::npos && name.find("KQ_mask") == std::string::npos) { - std::cout << *(tensor.data()) << std::endl; + std::cout << *(tensor.data()) << '\n'; } else { size_t rows = tensor.get_shape()[2]; size_t cols = tensor.get_shape()[3]; - auto * data = tensor.data(); + const float * data = tensor.data(); for (size_t i = 0; i < rows; ++i) { for (size_t j = 0; j < cols; ++j) { float val = data[i * cols + j]; @@ -1669,26 +1663,26 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor std::cout << std::setw(5) << val; } } - std::cout << std::endl; + std::cout << '\n'; } } break; } case ov::element::f16: - std::cout << *(tensor.data()) << std::endl; + std::cout << *(tensor.data()) << '\n'; break; case ov::element::i32: for (size_t i = 0; i < tensor.get_size(); ++i) { - std::cout << tensor.data()[i] << " "; + std::cout << tensor.data()[i] << ' '; } - std::cout << std::endl; + std::cout << '\n'; break; case ov::element::i64: for (size_t i = 0; i < tensor.get_size(); ++i) { - std::cout << tensor.data()[i] << " "; + std::cout << tensor.data()[i] << ' '; } - std::cout << std::endl; + std::cout << '\n'; break; default: break; @@ -1697,7 +1691,7 @@ void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor void print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst) { std::cout << "Output name: " << name << ", Output shape: " << tensor.get_shape() << ", Address: " << output_dst - << std::endl; + << '\n'; auto print_float_stats = [](const std::string & type_name, size_t size, auto get_value) { if (size == 0) { @@ -1711,20 +1705,16 @@ void print_output_tensor_info(const std::string & name, const ov::Tensor & tenso for (size_t i = 1; i < size; ++i) { float v = get_value(i); - if (v < min) { - min = v; - } - if (v > max) { - max = v; - } + min = std::min(v, min); + max = std::max(v, max); sum += v; } double mean = sum / size; std::cout << std::right << std::setw(6) << type_name << std::right << std::setw(12) << "First" << std::setw(12) - << "Min" << std::setw(12) << "Max" << std::setw(12) << "Mean" << std::endl; + << "Min" << std::setw(12) << "Max" << std::setw(12) << "Mean" << '\n'; std::cout << std::right << std::setw(6) << "" << std::right << std::setw(12) << first << std::setw(12) << min - << std::setw(12) << max << std::setw(12) << mean << std::endl; + << std::setw(12) << max << std::setw(12) << mean << '\n'; }; switch (tensor.get_element_type()) { @@ -1781,5 +1771,3 @@ int64_t get_inp_pos_n_tokens(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) bool get_is_prefill(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) { return get_inp_pos_n_tokens(cgraph, inp_pos) > 1; } - -#pragma GCC diagnostic pop diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 235b15d7e90a..74c25f0acea8 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -142,9 +142,6 @@ struct ov_runtime_context { enum ggml_status ov_graph_compute(struct ggml_cgraph * cgraph, ggml_backend_t backend); -enum ggml_status ov_graph_compute_dynamic(struct ggml_cgraph * cgraph, std::shared_ptr r_ctx); -enum ggml_status ov_graph_compute_static(struct ggml_cgraph * cgraph, std::shared_ptr r_ctx); - size_t checksum(const void * data, size_t size); bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path); @@ -185,18 +182,6 @@ int64_t get_inp_pos_n_tokens(struct ggml_cgraph * cgraph, const ggml_tensor * in bool get_is_prefill(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos); -ov::Tensor get_ov_input_tensor(std::shared_ptr ggml_decoder, const std::string & param_name); -ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr ggml_decoder, - const std::string & param_name); -ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr ggml_decoder, - const std::string & param_name, - int chunk_index); - -ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, - std::shared_ptr infer_request, - int output_index, - const ggml_tensor * ggml_tensor); - bool is_naive(struct ggml_cgraph * cgraph); /** @@ -205,9 +190,3 @@ bool is_naive(struct ggml_cgraph * cgraph); * @return true if the graph is identified as split; otherwise false. */ bool is_model_splitted(struct ggml_cgraph * cgraph); - -enum ggml_status naive_compute(struct ggml_cgraph * cgraph, - ov::Core & core, - const std::string & device, - const ov::AnyMap & config, - ov_compiled_model_cache & cache);