From 8c4e5473c1e070882d4362881deb5731e8bece69 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Tue, 21 Jul 2026 09:18:17 +0700 Subject: [PATCH 01/24] install OpenVINO runtime --- docs/backend/ov.md | 78 ++++++++++ scripts/install_ov.sh | 337 +++++++++++++++++++++++++++++++++++++++++ scripts/openvino.patch | 88 +++++++++++ 3 files changed, 503 insertions(+) create mode 100644 docs/backend/ov.md create mode 100644 scripts/install_ov.sh create mode 100644 scripts/openvino.patch diff --git a/docs/backend/ov.md b/docs/backend/ov.md new file mode 100644 index 0000000..48e1ea3 --- /dev/null +++ b/docs/backend/ov.md @@ -0,0 +1,78 @@ +# OpenVINO Backend for vla.cpp + +OpenVINO is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware, including CPUs, GPUs, and NPUs, in the cloud, on-premises, and on the edge. OpenVINO backend for vla.cpp enables hardware-accelerated inference on Intel® CPUs, GPUs, and NPUs while remaining compatible with the existing GGUF model ecosystem. The backend translates GGML compute graphs into OpenVINO graphs and leverages graph compilation, kernel fusion, and device-specific optimizations to improve inference performance on supported Intel hardware. + +## Supported Devices +OpenVINO backend supports the following hardware: +- Intel CPUs +- Intel GPUs (integrated and discrete) +- Intel NPUs + +## Prerequisites + +- Linux system (22.04 or 24.04) with Intel hardware (CPU, GPU, or NPU) +- For Intel GPU or NPU Usage: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md) +- OpenCL C++ headers, required to build the backend (the automatic install + script below installs them): + ```bash + sudo apt-get install opencl-clhpp-headers ocl-icd-opencl-dev + ``` + +## Install OpenVINO Runtime +- For manual installation, follow the guide to install OpenVINO Runtime from an archive file: [Install OpenVINO Runtime from an Archive File](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-linux.html) +- For automatic installation, run the following command: + ```bash + bash scripts/install_ov.sh + ``` + +## Build + +Initialize the OpenVINO environment, configure the OpenVINO GGML backend, patch +the fetched llama.cpp sources, and build the server: + +```bash +source /opt/intel/openvino/setupvars.sh +cmake -B build/ReleaseOV -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_OPENVINO=ON +git -C build/ReleaseOV/_deps/llama-src apply "scripts/openvino.patch" +cmake --build build/ReleaseOV --parallel +``` + +The patch makes the backend select the Intel OpenCL platform. Without it, on a +machine with more than one OpenCL runtime (e.g. an NVIDIA GPU next to the Intel +GPU), `GGML_OPENVINO_DEVICE=GPU` aborts at startup with "Incompatible OpenCL +runtime: program is not in expected ELF format". Configure fetches llama.cpp +before the patch step, so apply it again after a clean reconfigure. + +## Run + +Set `GGML_OPENVINO_DEVICE` to the actual target name. Do not enter the +documentation placeholder `` literally: in a shell, angle brackets +are interpreted as input redirection. + +```bash +GGML_OPENVINO_DEVICE=GPU \ +GGML_OPENVINO_STATEFUL_EXECUTION=1 \ +./build/ReleaseOV/vla-server ./weights/smolvla-libero.gguf +``` + +Use `CPU`, `GPU`, or `NPU` according to the devices exposed by the installed +OpenVINO runtime. At startup, two diagnostics identify the selection: + +```text +OpenVINO: using device GPU +vla: backend = OPENVINO (requested device GPU) +``` + +The first line is authoritative for the resolved OpenVINO device. If the +requested device is unavailable, the OpenVINO backend emits a warning, falls +back to CPU, and reports `OpenVINO: using device CPU`. The `vla:` line separately +confirms that GGML is using its OpenVINO backend rather than its native CPU +backend. + +OpenVINO compiles the model graphs on the first inference request (SmolVLA: +about 1 minute on CPU, 2-3 minutes on GPU). Use a client receive timeout above +this. Set `GGML_OPENVINO_CACHE_DIR=` to cache compiled graphs across +server restarts. + diff --git a/scripts/install_ov.sh b/scripts/install_ov.sh new file mode 100644 index 0000000..8fba84d --- /dev/null +++ b/scripts/install_ov.sh @@ -0,0 +1,337 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK_DIR="${SCRIPT_DIR}/.openvino_install_work" + +OS_ID="" +OS_VERSION="" + +log() { + printf '[install_openvino_runetime] %s\n' "$*" +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "Error: '$1' is required but not installed." >&2 + exit 1 + } +} + +detect_os() { + if [[ ! -f /etc/os-release ]]; then + echo "Error: /etc/os-release not found; cannot detect Ubuntu version." >&2 + exit 1 + fi + + # shellcheck disable=SC1091 + source /etc/os-release + + OS_ID="${ID:-}" + OS_VERSION="${VERSION_ID:-}" + + if [[ "${OS_ID}" != "ubuntu" ]]; then + echo "Error: this installer supports Ubuntu only. Detected ID='${OS_ID:-unknown}'." >&2 + exit 1 + fi +} + +prepare_common_tools() { + need_cmd bash + need_cmd sudo + need_cmd apt-get + need_cmd wget + need_cmd curl + need_cmd tar + need_cmd dpkg + need_cmd sha256sum + need_cmd find + need_cmd sort + + mkdir -p "${WORK_DIR}" +} + +prepare_common_dependencies() { + log "Installing common dependencies..." + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + libcurl4-openssl-dev \ + libtbb12 \ + cmake \ + ninja-build \ + python3-pip \ + curl \ + wget \ + tar \ + libopencl1 \ + ocl-icd-opencl-dev \ + opencl-headers \ + opencl-clhpp-headers \ + clinfo +} + +add_render_group() { + local target_user="${SUDO_USER:-$USER}" + if [[ -z "${target_user}" ]]; then + return + fi + + if id -nG "${target_user}" | grep -qw render; then + log "User ${target_user} is already in the render group." + else + log "Adding ${target_user} to render group..." + sudo gpasswd -a "${target_user}" render || true + log "Re-login (or restart shell session) to apply render group membership." + fi +} + +install_gpu_2204() { + local download_dir="${WORK_DIR}/intel_gpu_2204" + local igc_base_url="https://github.com/intel/intel-graphics-compiler/releases/download/v2.10.8" + local crt_base_url="https://github.com/intel/compute-runtime/releases/download/25.13.33276.16" + local checksum_file="ww13.sum" + local packages=( + "${igc_base_url}/intel-igc-core-2_2.10.8+18926_amd64.deb" + "${igc_base_url}/intel-igc-opencl-2_2.10.8+18926_amd64.deb" + "${crt_base_url}/intel-level-zero-gpu-dbgsym_1.6.33276.16_amd64.ddeb" + "${crt_base_url}/intel-level-zero-gpu_1.6.33276.16_amd64.deb" + "${crt_base_url}/intel-opencl-icd-dbgsym_25.13.33276.16_amd64.ddeb" + "${crt_base_url}/intel-opencl-icd_25.13.33276.16_amd64.deb" + "${crt_base_url}/libigdgmm12_22.7.0_amd64.deb" + ) + + log "Installing Intel GPU drivers for Ubuntu 22.04..." + mkdir -p "${download_dir}" + cd "${download_dir}" + + for url in "${packages[@]}"; do + wget -c "${url}" + done + wget -c "${crt_base_url}/${checksum_file}" + + sha256sum -c "${checksum_file}" + + shopt -s nullglob + local artifacts=( *.deb *.ddeb ) + if [[ ${#artifacts[@]} -eq 0 ]]; then + echo "Error: no GPU package files found for Ubuntu 22.04." >&2 + exit 1 + fi + + sudo dpkg -i "${artifacts[@]}" || sudo apt-get install -f -y + cd "${SCRIPT_DIR}" + rm -rf "${download_dir}" +} + +install_npu_2204() { + local download_dir="${WORK_DIR}/intel_npu_2204" + local npu_tarball="linux-npu-driver-v1.26.0.20251125-19665715237-ubuntu2204.tar.gz" + local npu_url="https://github.com/intel/linux-npu-driver/releases/download/v1.26.0/${npu_tarball}" + local level_zero_deb="level-zero_1.24.2+u22.04_amd64.deb" + local level_zero_url="https://github.com/oneapi-src/level-zero/releases/download/v1.24.2/${level_zero_deb}" + + log "Installing Intel NPU drivers for Ubuntu 22.04..." + sudo dpkg --purge --force-remove-reinstreq \ + intel-driver-compiler-npu \ + intel-fw-npu \ + intel-level-zero-npu \ + intel-level-zero-npu-dbgsym || true + + mkdir -p "${download_dir}" + cd "${download_dir}" + + wget -c "${npu_url}" + tar -xf "${npu_tarball}" + + mapfile -t npu_debs < <(find . -type f -name '*.deb' ! -name 'level-zero*.deb' | sort) + if [[ ${#npu_debs[@]} -eq 0 ]]; then + echo "Error: no Intel NPU .deb packages found for Ubuntu 22.04." >&2 + exit 1 + fi + + sudo dpkg -i "${npu_debs[@]}" || sudo apt-get install -f -y + + wget -c "${level_zero_url}" + sudo dpkg -i "${level_zero_deb}" || sudo apt-get install -f -y + + add_render_group + + cd "${SCRIPT_DIR}" + rm -rf "${download_dir}" +} + +install_runtime_2204() { + local download_dir="${WORK_DIR}/openvino_runtime_2204" + local openvino_version="${OPENVINO_VERSION:-2025.3}" + local openvino_build="${OPENVINO_BUILD:-19807.44526285f24}" + local openvino_archive="openvino_toolkit_ubuntu22_${openvino_version}.0.${openvino_build}_x86_64.tgz" + local openvino_dirname="openvino_toolkit_ubuntu22_${openvino_version}.0.${openvino_build}_x86_64" + local openvino_url="https://storage.openvinotoolkit.org/repositories/openvino/packages/${openvino_version}/linux/${openvino_archive}" + local install_root="${INSTALL_ROOT:-/opt/intel}" + local install_dir="${install_root}/openvino_${openvino_version}" + local symlink_path="${install_root}/openvino" + local archive_path="${download_dir}/openvino_${openvino_version}.tgz" + + log "Installing OpenVINO runtime for Ubuntu 22.04..." + sudo mkdir -p "${install_root}" + mkdir -p "${download_dir}" + + curl -fL "${openvino_url}" --output "${archive_path}" + rm -rf "${download_dir:?}/${openvino_dirname}" + tar -xf "${archive_path}" -C "${download_dir}" + + sudo rm -rf "${install_dir}" + sudo mv "${download_dir}/${openvino_dirname}" "${install_dir}" + sudo ln -sfn "openvino_${openvino_version}" "${symlink_path}" + + if [[ ! -f "${symlink_path}/setupvars.sh" ]]; then + echo "Error: ${symlink_path}/setupvars.sh was not found after installation." >&2 + exit 1 + fi + + rm -rf "${download_dir}" +} + +install_gpu_2404() { + local download_dir="${WORK_DIR}/intel_gpu_2404" + local igc_base_url="https://github.com/intel/intel-graphics-compiler/releases/download/v2.36.3" + local crt_base_url="https://github.com/intel/compute-runtime/releases/download/26.22.38646.4" + local checksum_file="ww22.sum" + local packages=( + "${igc_base_url}/intel-igc-core-2_2.36.3+21719_amd64.deb" + "${igc_base_url}/intel-igc-opencl-2_2.36.3+21719_amd64.deb" + "${crt_base_url}/intel-ocloc-dbgsym_26.22.38646.4-0_amd64.ddeb" + "${crt_base_url}/intel-ocloc_26.22.38646.4-0_amd64.deb" + "${crt_base_url}/intel-opencl-icd-dbgsym_26.22.38646.4-0_amd64.ddeb" + "${crt_base_url}/intel-opencl-icd_26.22.38646.4-0_amd64.deb" + "${crt_base_url}/libigdgmm12_22.10.0_amd64.deb" + "${crt_base_url}/libze-intel-gpu1-dbgsym_26.22.38646.4-0_amd64.ddeb" + "${crt_base_url}/libze-intel-gpu1_26.22.38646.4-0_amd64.deb" + ) + + log "Installing Intel GPU drivers for Ubuntu 24.04..." + mkdir -p "${download_dir}" + cd "${download_dir}" + + for url in "${packages[@]}"; do + wget -c "${url}" + done + wget -c "${crt_base_url}/${checksum_file}" + + sha256sum -c "${checksum_file}" + + shopt -s nullglob + local artifacts=( *.deb *.ddeb ) + if [[ ${#artifacts[@]} -eq 0 ]]; then + echo "Error: no GPU package files found for Ubuntu 24.04." >&2 + exit 1 + fi + + sudo dpkg -i "${artifacts[@]}" || sudo apt-get install -f -y + cd "${SCRIPT_DIR}" + rm -rf "${download_dir}" +} + +install_npu_2404() { + local download_dir="${WORK_DIR}/intel_npu_2404" + local npu_release="v1.33.0" + local npu_archive="linux-npu-driver-v1.33.0.20260529-26625960453-ubuntu2404.tar.gz" + local npu_url="https://github.com/intel/linux-npu-driver/releases/download/${npu_release}/${npu_archive}" + local npu_packages=( + intel-driver-compiler-npu + intel-fw-npu + intel-level-zero-npu + intel-level-zero-npu-dbgsym + ) + + log "Installing Intel NPU drivers for Ubuntu 24.04..." + sudo dpkg --purge --force-remove-reinstreq "${npu_packages[@]}" || true + + mkdir -p "${download_dir}" + cd "${download_dir}" + + wget -c "${npu_url}" + tar -xf "${npu_archive}" + + shopt -s nullglob + local debs=( *.deb ) + if [[ ${#debs[@]} -eq 0 ]]; then + echo "Error: no Intel NPU .deb packages found for Ubuntu 24.04." >&2 + exit 1 + fi + + sudo dpkg -i "${debs[@]}" || sudo apt-get install -f -y + + add_render_group + + cd "${SCRIPT_DIR}" + rm -rf "${download_dir}" +} + +install_runtime_2404() { + local download_dir="${WORK_DIR}/openvino_runtime_2404" + local openvino_version="${OPENVINO_VERSION:-2026.2.1}" + local openvino_build="${OPENVINO_BUILD:-21919.ede283a88e3}" + local openvino_archive="openvino_toolkit_ubuntu24_${openvino_version}.${openvino_build}_x86_64.tgz" + local openvino_dirname="openvino_toolkit_ubuntu24_${openvino_version}.${openvino_build}_x86_64" + local openvino_url="https://storage.openvinotoolkit.org/repositories/openvino/packages/${openvino_version}/linux/${openvino_archive}" + local install_root="${INSTALL_ROOT:-/opt/intel}" + local install_dir="${install_root}/openvino_${openvino_version}" + local symlink_path="${install_root}/openvino" + local archive_path="${download_dir}/openvino_${openvino_version}.tgz" + + log "Installing OpenVINO runtime for Ubuntu 24.04..." + sudo mkdir -p "${install_root}" + mkdir -p "${download_dir}" + + curl -fL "${openvino_url}" --output "${archive_path}" + rm -rf "${download_dir:?}/${openvino_dirname}" + tar -xf "${archive_path}" -C "${download_dir}" + + sudo rm -rf "${install_dir}" + sudo mv "${download_dir}/${openvino_dirname}" "${install_dir}" + sudo ln -sfn "openvino_${openvino_version}" "${symlink_path}" + + if [[ ! -f "${symlink_path}/setupvars.sh" ]]; then + echo "Error: ${symlink_path}/setupvars.sh was not found after installation." >&2 + exit 1 + fi + + rm -rf "${download_dir}" +} + +run_installation() { + case "${OS_VERSION}" in + 22.04) + install_gpu_2204 + install_npu_2204 + install_runtime_2204 + ;; + 24.04) + install_gpu_2404 + install_npu_2404 + install_runtime_2404 + ;; + *) + echo "Error: unsupported Ubuntu version '${OS_VERSION:-unknown}'. Supported versions: 22.04, 24.04." >&2 + exit 1 + ;; + esac +} + +main() { + detect_os + prepare_common_tools + prepare_common_dependencies + + log "Detected Ubuntu ${OS_VERSION}." + run_installation + rm -rf "${WORK_DIR}" + + log "All OpenVINO installation steps completed successfully." + log "To load OpenVINO in current shell: source /opt/intel/openvino/setupvars.sh" +} + +main "$@" diff --git a/scripts/openvino.patch b/scripts/openvino.patch new file mode 100644 index 0000000..58e34d8 --- /dev/null +++ b/scripts/openvino.patch @@ -0,0 +1,88 @@ +diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +index d9ad7be73..75d34beb4 100644 +--- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp ++++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +@@ -9,6 +9,7 @@ + #include + #include + #include ++#include + + ov::Core & ov_singleton_core() { + static ov::Core core; +@@ -19,6 +20,39 @@ ov::Core & ov_singleton_core() { + // Device Configuration Implementations + // ===================================================== + ++// Find the Intel OpenCL platform. With multiple OpenCL runtimes installed ++// (e.g. NVIDIA + Intel), the first platform is not always the Intel one, ++// and the OpenVINO GPU plugin only accepts Intel contexts. ++static cl_platform_id ggml_openvino_get_intel_platform() { ++ static cl_platform_id platform = nullptr; ++ static bool searched = false; ++ if (searched) { ++ return platform; ++ } ++ searched = true; ++ ++ cl_uint n_platforms = 0; ++ if (clGetPlatformIDs(0, nullptr, &n_platforms) != CL_SUCCESS || n_platforms == 0) { ++ return nullptr; ++ } ++ std::vector platforms(n_platforms); ++ if (clGetPlatformIDs(n_platforms, platforms.data(), nullptr) != CL_SUCCESS) { ++ return nullptr; ++ } ++ ++ for (cl_platform_id p : platforms) { ++ char vendor[256] = ""; ++ if (clGetPlatformInfo(p, CL_PLATFORM_VENDOR, sizeof(vendor), vendor, nullptr) != CL_SUCCESS) { ++ continue; ++ } ++ if (strstr(vendor, "Intel") != nullptr) { ++ platform = p; ++ break; ++ } ++ } ++ return platform; ++} ++ + void ggml_openvino_device_config::init() { + if (initialized) { + return; +@@ -88,10 +122,9 @@ void ggml_openvino_device_config::init() { + if (device_name == "GPU") { + // Create OpenCL context and queue + cl_int err; +- cl_platform_id platform; +- err = clGetPlatformIDs(1, &platform, nullptr); +- if (err != CL_SUCCESS) { +- GGML_LOG_ERROR("Failed to get OpenCL platform: %d\n", err); ++ cl_platform_id platform = ggml_openvino_get_intel_platform(); ++ if (platform == nullptr) { ++ GGML_LOG_ERROR("Failed to find an Intel OpenCL platform\n"); + return; + } + +@@ -194,8 +227,8 @@ clEnqueueMemFillINTEL_fn ggml_openvino_get_clEnqueueMemFillINTEL() { + static bool loaded = false; + if (!loaded) { + loaded = true; +- cl_platform_id platform; +- if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) { ++ cl_platform_id platform = ggml_openvino_get_intel_platform(); ++ if (platform != nullptr) { + fn = (clEnqueueMemFillINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueMemFillINTEL"); + } + } +@@ -208,8 +241,8 @@ clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL() { + static bool loaded = false; + if (!loaded) { + loaded = true; +- cl_platform_id platform; +- if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) { ++ cl_platform_id platform = ggml_openvino_get_intel_platform(); ++ if (platform != nullptr) { + fn = (clEnqueueMemcpyINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueMemcpyINTEL"); + } + } From f079b8a3d21417584f533911b3992258808ec503 Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 11:14:21 +0700 Subject: [PATCH 02/24] add an OpenVINO backend build and carry the vision towers through ggml-openvino --- CMakeLists.txt | 23 +++- README.md | 6 + docs/backend/ov.md | 201 ++++++++++++++++++++++++-------- scripts/openvino.patch | 88 -------------- scripts/patch_ggml_openvino.py | 204 +++++++++++++++++++++++++++++++++ src/backend.h | 101 +++++++++++++++- src/loader.cpp | 6 +- src/models/bitvla.cpp | 5 + src/models/evo1.cpp | 2 + src/models/gr00tn1d5.cpp | 2 + src/models/gr00tn1d6.cpp | 3 + src/models/gr00tn1d7.cpp | 2 + src/models/openvla_oft.cpp | 2 + src/models/pi0.cpp | 2 + src/models/pi05.cpp | 2 + src/models/smolvla.cpp | 16 ++- src/models/vla_adapter.cpp | 2 + src/models/vla_jepa.cpp | 3 + 18 files changed, 522 insertions(+), 148 deletions(-) delete mode 100644 scripts/openvino.patch create mode 100755 scripts/patch_ggml_openvino.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 89da1ee..d97a2cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ if(NOT CMAKE_BUILD_TYPE) endif() set(_vla_accel "") -foreach(_flag GGML_CUDA GGML_SYCL GGML_METAL) +foreach(_flag GGML_CUDA GGML_SYCL GGML_METAL GGML_OPENVINO) if(${_flag}) list(APPEND _vla_accel ${_flag}) endif() @@ -35,10 +35,15 @@ if(GGML_CUDA) find_package(Python3 COMPONENTS Interpreter REQUIRED) set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_cuda_ext_hook.py ) +elseif(GGML_OPENVINO) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_openvino.py ) endif() # Overridable so a regression can be bisected against another tag in a separate # build dir (-DVLA_LLAMA_TAG=b10326) without editing this file. The patch -# anchors in scripts/patch_ggml_cuda_ext_hook.py are checked against the default. +# anchors in scripts/patch_ggml_cuda_ext_hook.py and scripts/patch_ggml_openvino.py +# are checked against the default. set(VLA_LLAMA_TAG "b10331" CACHE STRING "llama.cpp tag to fetch") include(FetchContent) @@ -173,6 +178,20 @@ if(GGML_METAL AND NOT GGML_CUDA AND NOT GGML_SYCL) target_compile_definitions(vla_core PUBLIC GGML_USE_METAL) endif() +if(GGML_OPENVINO AND NOT GGML_CUDA AND NOT GGML_SYCL AND NOT GGML_METAL) + # ggml's own OpenVINO backend target finds the toolkit; this only tells the + # archs which branch of the backend.h ladder to compile. Fail early with a + # pointer to setupvars.sh rather than deep inside the fetched tree. + find_package(OpenVINO QUIET COMPONENTS Runtime) + if(NOT OpenVINO_FOUND) + message(FATAL_ERROR + "GGML_OPENVINO=ON but the OpenVINO runtime was not found. Install it " + "(scripts/install_ov.sh) and 'source /opt/intel/openvino/setupvars.sh' " + "in the shell that configures. See docs/backend/ov.md.") + endif() + target_compile_definitions(vla_core PUBLIC GGML_USE_OPENVINO) +endif() + add_library(vlm_core src/vlm/engine.cpp ) diff --git a/README.md b/README.md index 4f06b44..ed0c282 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ from consumer GPUs down to Jetson-class boards - or **Intel GPUs** via SYCL. - CUDA 12.x (optional - required only for CUDA GPU builds) - Intel oneAPI 2025.x + GPU compute runtime (optional - only for Intel GPU builds, see [docs/backend/sycl.md](docs/backend/sycl.md)) +- OpenVINO 2026.x runtime (optional - only for the in-progress OpenVINO backend, + see [docs/backend/ov.md](docs/backend/ov.md)) - `libzmq3-dev`, `cppzmq-dev`, `libprotobuf-dev`, `protobuf-compiler` ```bash @@ -309,6 +311,10 @@ Experimental results on other platforms can be found in Support matrix of models (rows) against platforms (columns). Legend: `Y` = supported (released and benchmarked), `~` = in progress, `-` = planned. +OpenVINO builds and selects its backend today, but no arch completes a +prediction yet - the remaining blocker is upstream in ggml's OpenVINO backend, +written up in [docs/backend/ov.md](docs/backend/ov.md). + | Model | CPU (x86-64 / ARM) | CUDA | SYCL (Intel) | Metal | OpenVINO | |---|:--:|:--:|:--:|:--:|:--:| | [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | - | diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 48e1ea3..20f2a37 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -1,78 +1,185 @@ -# OpenVINO Backend for vla.cpp +# `vla.cpp` on Intel CPUs, GPUs and NPUs (OpenVINO backend) -OpenVINO is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware, including CPUs, GPUs, and NPUs, in the cloud, on-premises, and on the edge. OpenVINO backend for vla.cpp enables hardware-accelerated inference on Intel® CPUs, GPUs, and NPUs while remaining compatible with the existing GGUF model ecosystem. The backend translates GGML compute graphs into OpenVINO graphs and leverages graph compilation, kernel fusion, and device-specific optimizations to improve inference performance on supported Intel hardware. +Notes for building `vla.cpp` against ggml's OpenVINO backend, and an honest +account of how far it currently runs. Like SYCL, OpenVINO is **not** +auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO +runtime on the configure line. + +> **Status: builds and runs, no arch completes a prediction yet.** The backend +> comes up, weights fold in, and the vision towers translate and execute. The +> language model and action expert do not: ggml's OpenVINO backend models a +> decoder-only LLM with one position input and an F16 KV cache, and every +> vla.cpp arch has several position inputs and no KV cache. See +> [What still blocks it](#what-still-blocks-it). The OpenVINO column of the +> README support matrix stays `-` until an arch passes end to end. + +OpenVINO is Intel's inference toolkit; ggml's backend translates a ggml compute +graph into an OpenVINO model and hands it to the CPU, GPU or NPU plugin, which +compiles and fuses it for the device. Unlike SYCL it needs no separate compiler: +the stock GCC/Clang build links `libopenvino` and everything else is ordinary +C++. + +## Supported devices -## Supported Devices -OpenVINO backend supports the following hardware: - Intel CPUs -- Intel GPUs (integrated and discrete) -- Intel NPUs +- Intel GPUs (integrated Xe / Arc, and discrete) +- Intel NPUs (Core Ultra) ## Prerequisites -- Linux system (22.04 or 24.04) with Intel hardware (CPU, GPU, or NPU) -- For Intel GPU or NPU Usage: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md) -- OpenCL C++ headers, required to build the backend (the automatic install - script below installs them): - ```bash - sudo apt-get install opencl-clhpp-headers ocl-icd-opencl-dev - ``` +Linux (Ubuntu 22.04 or 24.04) on Intel hardware. + +### 1. Device access + +The CPU plugin needs nothing. The GPU and NPU plugins reach the hardware through +`/dev/dri/renderD*` and `/dev/accel/accel0`, both owned by the `render` group: + +```bash +sudo usermod -aG render,video "$USER" +``` + +Re-login, then check that the OpenCL runtime actually enumerates the GPU: + +```bash +clinfo -l +``` + +`Number of platforms 0` with `/etc/OpenCL/vendors/intel.icd` present almost +always means the render group has not taken effect yet. Without it, +`GGML_OPENVINO_DEVICE=GPU` warns and silently falls back to the CPU plugin. + +For the GPU compute runtime and NPU driver packages themselves, follow +[llama.cpp's OpenVINO notes](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md). -## Install OpenVINO Runtime -- For manual installation, follow the guide to install OpenVINO Runtime from an archive file: [Install OpenVINO Runtime from an Archive File](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-linux.html) -- For automatic installation, run the following command: - ```bash - bash scripts/install_ov.sh - ``` +### 2. OpenVINO runtime + OpenCL headers -## Build +```bash +sudo apt-get install -y opencl-clhpp-headers ocl-icd-opencl-dev opencl-headers +``` + +Then either install OpenVINO +[from the archive](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-linux.html) +by hand, or run the bundled installer, which also pulls the GPU driver stack and +adds you to `render`: + +```bash +bash scripts/install_ov.sh +``` + +Plus the usual host dependencies: -Initialize the OpenVINO environment, configure the OpenVINO GGML backend, patch -the fetched llama.cpp sources, and build the server: +```bash +sudo apt-get install -y cmake ninja-build pkg-config \ + protobuf-compiler libprotobuf-dev libzmq3-dev cppzmq-dev +``` + +## Configure & build ```bash source /opt/intel/openvino/setupvars.sh -cmake -B build/ReleaseOV -G Ninja \ + +cmake -B build-ov -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DGGML_OPENVINO=ON -git -C build/ReleaseOV/_deps/llama-src apply "scripts/openvino.patch" -cmake --build build/ReleaseOV --parallel +cmake --build build-ov -j$(nproc) ``` -The patch makes the backend select the Intel OpenCL platform. Without it, on a -machine with more than one OpenCL runtime (e.g. an NVIDIA GPU next to the Intel -GPU), `GGML_OPENVINO_DEVICE=GPU` aborts at startup with "Incompatible OpenCL -runtime: program is not in expected ELF format". Configure fetches llama.cpp -before the patch step, so apply it again after a clean reconfigure. +`setvars`-style sourcing is needed in every shell that builds *or* runs the +binaries: `libopenvino.so` and its TBB live under `/opt/intel`. Configure fails +early with a pointer back here if the runtime is not on `CMAKE_PREFIX_PATH`. + +`scripts/patch_ggml_openvino.py` runs as the FetchContent patch step, so the +four ggml fixes described in its docstring are applied automatically and +re-applied on a clean reconfigure. There is no manual `git apply`. ## Run -Set `GGML_OPENVINO_DEVICE` to the actual target name. Do not enter the -documentation placeholder `` literally: in a shell, angle brackets -are interpreted as input redirection. +`GGML_OPENVINO_DEVICE` picks the target by name. Do not type the placeholder +`` literally - in a shell the angle brackets are input redirection. ```bash -GGML_OPENVINO_DEVICE=GPU \ -GGML_OPENVINO_STATEFUL_EXECUTION=1 \ -./build/ReleaseOV/vla-server ./weights/smolvla-libero.gguf +GGML_OPENVINO_DEVICE=GPU ./build-ov/vla-server ./weights/smolvla-libero.gguf ``` -Use `CPU`, `GPU`, or `NPU` according to the devices exposed by the installed -OpenVINO runtime. At startup, two diagnostics identify the selection: +Two lines identify the selection at startup: ```text OpenVINO: using device GPU vla: backend = OPENVINO (requested device GPU) ``` -The first line is authoritative for the resolved OpenVINO device. If the -requested device is unavailable, the OpenVINO backend emits a warning, falls -back to CPU, and reports `OpenVINO: using device CPU`. The `vla:` line separately -confirms that GGML is using its OpenVINO backend rather than its native CPU -backend. +The first comes from ggml and is authoritative: an unavailable device logs a +warning there and falls back to `CPU`, which is still the OpenVINO CPU plugin, +not ggml's native CPU backend. The second line echoes what was requested, so the +pair tells you whether you got the device you asked for. `VLA_DEVICE` does *not* +apply - ggml exposes OpenVINO as a single device and the target is chosen by +name. + +OpenVINO compiles each graph on first use, which is slow (minutes for a vision +tower). Set `GGML_OPENVINO_CACHE_DIR=` to keep compiled graphs across +restarts, and give any client a receive timeout well above the first request. + +## What vla.cpp had to change + +Three of these are ordinary correctness fixes that happen to be invisible on the +other backends: + +- **Weight buffers are tagged.** `ggml_backend_alloc_ctx_tensors` leaves a + buffer on `GGML_BACKEND_BUFFER_USAGE_ANY`, and ggml-openvino reads ANY as "KV + cache", giving every weight a dynamic sequence dimension. `vla::alloc_weights` + in [`src/backend.h`](../../src/backend.h) tags it `..._WEIGHTS`, which is what + llama.cpp does with its own weights and what lets the frontend fold them in as + constants. +- **Graph tensors get unique names.** ggml derives a result's name from its + source, so `ggml_reshape_2d` of an unnamed tensor is called `" (reshaped)"` - + and a graph whose intermediates were never named ends up with many tensors + sharing one name. ggml-openvino keys its translation map on those names, so + duplicates silently collapse into one node and the graph wires up the wrong + tensor. `vla::graph_unique_names` relabels duplicates before compute. It + compiles to nothing outside an OpenVINO build. +- **SmolVLA's time tiles moved out of the weight buffer.** They are precomputed + once but they are graph inputs, not checkpoint parameters. As weights they + became 2-D constants that could not be concatenated with the 4-D activation + beside them. +- **`GGML_OPENVINO_NAIVE_GRAPH_SIZE` defaults high.** ggml-openvino translates a + graph under 20 nodes literally and sends anything larger through an LLM model + builder. The literal path is the one that fits a vision tower; the threshold is + raised in `backend_init`, and an explicit setting still wins. + +None of it changes what the other backends compute: `vla_predict_check` on a CPU +build of this branch is byte-identical to the same build of `main` for SmolVLA +and π0.5, apart from the `weight_buf` line, which drops by the size of the time +tiles that moved. + +## What still blocks it + +With the above in place, SmolVLA's SigLIP tower translates and runs, and the +prefix/expert graph reaches OpenVINO's shape inference before failing: -OpenVINO compiles the model graphs on the first inference request (SmolVLA: -about 1 minute on CPU, 2-3 minutes on GPU). Use a client receive timeout above -this. Set `GGML_OPENVINO_CACHE_DIR=` to cache compiled graphs across -server restarts. +```text +opset1::Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32]) +Argument shapes are inconsistent. +``` +The two operands are RoPE tables of different lengths. `GgmlOvDecoder` maps +*every* tensor feeding a `GGML_OP_ROPE`'s second input to one graph parameter +named `inp_pos`, because an llama.cpp graph has exactly one position input. Every +vla.cpp arch has several - SmolVLA alone passes a prefill, a full and a rebased +position tensor - and they collapse onto each other. + +π0.5 fails on the same node with the same message (`[1,50,1,128]` against +`[1,262,1,128]`), so this is the shared blocker rather than a SmolVLA quirk. + +That is not something vla.cpp can work around from the outside: the fix belongs +in ggml-openvino, which needs to key position inputs per tensor rather than by a +fixed name. The same class of assumption shows up in the KV-cache-shaped dynamic +sequence dimension and in the `compute_op_case` pattern tables, two of which +already needed narrowing (see `scripts/patch_ggml_openvino.py`). + +Separately, several archs use ops the backend has no translator for at all - +`GGML_UNARY_OP_RELU` (every GR00T, Evo-1, VLA-Adapter, OpenVLA-OFT, BitVLA, +VLA-JEPA), `GGML_UNARY_OP_GELU_ERF`, `GGML_OP_NEG`, `GGML_OP_SQR` - and the core +drives a single backend through `gallocr` rather than a scheduler, so there is no +per-op CPU fallback to absorb them. SmolVLA, π0 and π0.5 are the three archs +whose op sets are fully covered today, which is why SmolVLA is the one to retest +first when the position-input handling lands upstream. diff --git a/scripts/openvino.patch b/scripts/openvino.patch deleted file mode 100644 index 58e34d8..0000000 --- a/scripts/openvino.patch +++ /dev/null @@ -1,88 +0,0 @@ -diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp -index d9ad7be73..75d34beb4 100644 ---- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp -+++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp -@@ -9,6 +9,7 @@ - #include - #include - #include -+#include - - ov::Core & ov_singleton_core() { - static ov::Core core; -@@ -19,6 +20,39 @@ ov::Core & ov_singleton_core() { - // Device Configuration Implementations - // ===================================================== - -+// Find the Intel OpenCL platform. With multiple OpenCL runtimes installed -+// (e.g. NVIDIA + Intel), the first platform is not always the Intel one, -+// and the OpenVINO GPU plugin only accepts Intel contexts. -+static cl_platform_id ggml_openvino_get_intel_platform() { -+ static cl_platform_id platform = nullptr; -+ static bool searched = false; -+ if (searched) { -+ return platform; -+ } -+ searched = true; -+ -+ cl_uint n_platforms = 0; -+ if (clGetPlatformIDs(0, nullptr, &n_platforms) != CL_SUCCESS || n_platforms == 0) { -+ return nullptr; -+ } -+ std::vector platforms(n_platforms); -+ if (clGetPlatformIDs(n_platforms, platforms.data(), nullptr) != CL_SUCCESS) { -+ return nullptr; -+ } -+ -+ for (cl_platform_id p : platforms) { -+ char vendor[256] = ""; -+ if (clGetPlatformInfo(p, CL_PLATFORM_VENDOR, sizeof(vendor), vendor, nullptr) != CL_SUCCESS) { -+ continue; -+ } -+ if (strstr(vendor, "Intel") != nullptr) { -+ platform = p; -+ break; -+ } -+ } -+ return platform; -+} -+ - void ggml_openvino_device_config::init() { - if (initialized) { - return; -@@ -88,10 +122,9 @@ void ggml_openvino_device_config::init() { - if (device_name == "GPU") { - // Create OpenCL context and queue - cl_int err; -- cl_platform_id platform; -- err = clGetPlatformIDs(1, &platform, nullptr); -- if (err != CL_SUCCESS) { -- GGML_LOG_ERROR("Failed to get OpenCL platform: %d\n", err); -+ cl_platform_id platform = ggml_openvino_get_intel_platform(); -+ if (platform == nullptr) { -+ GGML_LOG_ERROR("Failed to find an Intel OpenCL platform\n"); - return; - } - -@@ -194,8 +227,8 @@ clEnqueueMemFillINTEL_fn ggml_openvino_get_clEnqueueMemFillINTEL() { - static bool loaded = false; - if (!loaded) { - loaded = true; -- cl_platform_id platform; -- if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) { -+ cl_platform_id platform = ggml_openvino_get_intel_platform(); -+ if (platform != nullptr) { - fn = (clEnqueueMemFillINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueMemFillINTEL"); - } - } -@@ -208,8 +241,8 @@ clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL() { - static bool loaded = false; - if (!loaded) { - loaded = true; -- cl_platform_id platform; -- if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) { -+ cl_platform_id platform = ggml_openvino_get_intel_platform(); -+ if (platform != nullptr) { - fn = (clEnqueueMemcpyINTEL_fn) clGetExtensionFunctionAddressForPlatform(platform, "clEnqueueMemcpyINTEL"); - } - } diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py new file mode 100755 index 0000000..20efeac --- /dev/null +++ b/scripts/patch_ggml_openvino.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Four small fixes to the fetched ggml OpenVINO backend. + +ggml-openvino is written against llama.cpp's graphs: one decoder-only +transformer, one position input, an F16 KV cache. vla.cpp drives it with vision +towers and action experts instead, which is legal ggml but nothing the backend +has seen. Each hunk below is a place where an llama.cpp-shaped assumption is +narrower than the ggml contract. They are what carries a vla.cpp vision tower +through translation; see docs/backend/ov.md for what still does not. + + 1. ggml-openvino-extra.cpp - pick the *Intel* OpenCL platform. + `GGML_OPENVINO_DEVICE=GPU` builds an OpenVINO remote context on an OpenCL + queue and takes the first platform the ICD loader reports. With more than + one runtime installed (an NVIDIA card next to the Intel iGPU, POCL, + Rusticl) that is whichever `/etc/OpenCL/vendors/*.icd` sorted first, and + the GPU plugin only accepts an Intel context -- it aborts at startup with + "Incompatible OpenCL runtime: program is not in expected ELF format". + Selecting by `CL_PLATFORM_VENDOR` leaves single-runtime boxes unchanged. + + 2. ggml-decoder.cpp - narrow RESHAPE op_case 3. + Case 3 is the KV-cache flatten, `[512,1024,1,1] -> [1,524288,1,1]`, and it + emits a shape with `-1` in dim 2 and 1 in dim 3. Its guard only tests + `src->ne[0]*ne[1]*ne[2] == node->ne[1]`, which also matches the kernel + reshape inside `ggml_conv_2d` (`[16,16,3,768] -> [768,768]`) and mangles + it. The real case always has `node->ne[0] == 1`; requiring that sends the + conv kernel to case 6, the plain reshape. + + 3. openvino/op/flash_attn_ext.cpp - convert K/V to F16 with Q. + The translator converts Q, the mask and the scale to F16 because + llama.cpp's KV cache already is. vla.cpp keeps K/V in F32, and OpenVINO's + SDPA rejects mixed input types ("Mixed input types are not supported"). + Converting K/V too matches the precision the translator already chose. + + 4. utils.cpp - make the naive-path graph-size threshold settable. + Graphs under 20 nodes bypass the LLM decoder and translate literally, with + static shapes and no KV-cache inference. That literal path is the one that + suits a vision tower, but a vision tower is ~450 nodes. The constant + becomes `GGML_OPENVINO_NAIVE_GRAPH_SIZE`; src/backend.h defaults it high + for vla.cpp and an explicit setting still wins. + +Idempotent - re-running on a patched tree is a no-op, so a reconfigure that +re-populates the FetchContent source dir is safe either way. + +Usage: scripts/patch_ggml_openvino.py [] +""" + +import pathlib +import sys + +MARKER = "vla.cpp:" + +HELPER = """// vla.cpp: select the Intel OpenCL platform. With several OpenCL runtimes +// installed the first platform is not always Intel's, and the OpenVINO GPU +// plugin only accepts an Intel context. Cached: the ICD list cannot change +// under a running process. +static cl_platform_id ggml_openvino_get_intel_platform() { + static cl_platform_id platform = nullptr; + static bool searched = false; + if (searched) { + return platform; + } + searched = true; + + cl_uint n_platforms = 0; + if (clGetPlatformIDs(0, nullptr, &n_platforms) != CL_SUCCESS || n_platforms == 0) { + return nullptr; + } + std::vector platforms(n_platforms); + if (clGetPlatformIDs(n_platforms, platforms.data(), nullptr) != CL_SUCCESS) { + return nullptr; + } + + for (cl_platform_id p : platforms) { + char vendor[256] = ""; + if (clGetPlatformInfo(p, CL_PLATFORM_VENDOR, sizeof(vendor), vendor, nullptr) != CL_SUCCESS) { + continue; + } + if (strstr(vendor, "Intel") != nullptr) { + platform = p; + break; + } + } + return platform; +} + +""" + +USM_LOOKUP = """ cl_platform_id platform; + if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) { + fn = (%s_fn) clGetExtensionFunctionAddressForPlatform(platform, "%s"); +""" + +USM_LOOKUP_NEW = """ cl_platform_id platform = ggml_openvino_get_intel_platform(); + if (platform != nullptr) { + fn = (%s_fn) clGetExtensionFunctionAddressForPlatform(platform, "%s"); +""" + +# file -> [(anchor, replacement), ...]. Every anchor must match exactly once. +EDITS = { + "ggml/src/ggml-openvino/ggml-openvino-extra.cpp": [ + ("#include \n", "#include \n#include \n"), + ("void ggml_openvino_device_config::init() {", HELPER + "void ggml_openvino_device_config::init() {"), + ( + """ cl_int err; + cl_platform_id platform; + err = clGetPlatformIDs(1, &platform, nullptr); + if (err != CL_SUCCESS) { + GGML_LOG_ERROR("Failed to get OpenCL platform: %d\\n", err); + return; + } +""", + """ cl_int err; + cl_platform_id platform = ggml_openvino_get_intel_platform(); + if (platform == nullptr) { + GGML_LOG_ERROR("Failed to find an Intel OpenCL platform\\n"); + return; + } +""", + ), + (USM_LOOKUP % (("clEnqueueMemFillINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemFillINTEL",) * 2)), + (USM_LOOKUP % (("clEnqueueMemcpyINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemcpyINTEL",) * 2)), + ], + "ggml/src/ggml-openvino/ggml-decoder.cpp": [ + ( + """ } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { + op_case = 3;""", + """ // vla.cpp: case 3 is the KV-cache flatten, whose result is always + // [1, n, 1, 1]. Without the ne[0] test it also swallows the kernel + // reshape ggml_conv_2d emits and rewrites it to the wrong shape. + } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1] && node->ne[0] == 1) { + op_case = 3;""", + ), + ], + "ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp": [ + ( + """ auto q = std::make_shared(q_f32, ov::element::f16);""", + """ auto q = std::make_shared(q_f32, ov::element::f16); + // vla.cpp: Q, the mask and the scale below are all forced to F16 because + // llama.cpp's KV cache already is. K/V that arrive as F32 have to come along + // or SDPA rejects the mix. + if (k.get_element_type() != ov::element::f16) { + k = std::make_shared(k, ov::element::f16); + } + if (v.get_element_type() != ov::element::f16) { + v = std::make_shared(v, ov::element::f16); + }""", + ), + ], + "ggml/src/ggml-openvino/utils.cpp": [ + ( + """bool is_naive(ggml_cgraph * cgraph) { + constexpr int naive_graph_size_threshold = 20;""", + """bool is_naive(ggml_cgraph * cgraph) { + // vla.cpp: the literal translation path suits any graph that is not a + // decoder-only LLM, so let the caller raise the bar it is chosen under. + static const int naive_graph_size_threshold = [] { + const char * env = getenv("GGML_OPENVINO_NAIVE_GRAPH_SIZE"); + return (env && *env) ? atoi(env) : 20; + }();""", + ), + ], +} + + +def main() -> int: + root = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else ".") + + for rel, edits in EDITS.items(): + src = root / rel + if not src.is_file(): + print(f"patch_ggml_openvino: {src} not found", file=sys.stderr) + return 1 + text = src.read_text() + if MARKER in text: + print(f"patch_ggml_openvino: {rel} already patched") + continue + for anchor, replacement in edits: + n = text.count(anchor) + if n != 1: + print(f"patch_ggml_openvino: anchor matched {n} times in {rel}, expected 1:\n{anchor}", + file=sys.stderr) + return 1 + text = text.replace(anchor, replacement) + src.write_text(text) + print(f"patch_ggml_openvino: patched {rel}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/backend.h b/src/backend.h index 33377d4..3573495 100644 --- a/src/backend.h +++ b/src/backend.h @@ -21,10 +21,10 @@ * instead; the ladder lives here once. * * Exactly one accelerator is compiled in, picked by the CMake flag that was - * used (`GGML_CUDA` / `GGML_SYCL` / `GGML_METAL`). There is no per-op CPU - * fallback: the core drives a single backend through `gallocr` rather than a - * scheduler, so an arch that hits an op the backend does not implement asserts - * at predict time instead of silently limping. + * used (`GGML_CUDA` / `GGML_SYCL` / `GGML_METAL` / `GGML_OPENVINO`). There is no + * per-op CPU fallback: the core drives a single backend through `gallocr` rather + * than a scheduler, so an arch that hits an op the backend does not implement + * asserts at predict time instead of silently limping. */ #pragma once @@ -42,17 +42,24 @@ #ifdef GGML_USE_METAL #include "ggml-metal.h" #endif +#ifdef GGML_USE_OPENVINO +#include "ggml-openvino.h" +#endif #include #include -#ifdef GGML_USE_SYCL +#ifdef GGML_USE_OPENVINO +#include +#include +#endif +#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) #include // setenv / _putenv_s #include #endif namespace vla { -#ifdef GGML_USE_SYCL +#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) // setenv is POSIX. _putenv_s has no "do not overwrite" mode, so check first. inline void setenv_default(const char * key, const char * val) { #ifdef _WIN32 @@ -76,6 +83,59 @@ struct Backend { bool is_cuda = false; }; +/** + * @brief Give every tensor in a built graph a name unique within that graph. + * + * ggml derives a name for a result from its source -- `ggml_reshape_2d` of an + * unnamed tensor is called " (reshaped)" -- so a graph whose intermediates were + * never named ends up with many tensors sharing one name. That is legal ggml, + * and llama.cpp never trips over it because it labels every node it builds. + * + * ggml-openvino keys its translation map on those names: two nodes with the same + * name silently become one, and the graph it hands OpenVINO wires the wrong + * tensor into the next op. Renaming duplicates in place before compute is enough + * to keep them apart, and only the OpenVINO build pays for it -- elsewhere this + * compiles to nothing, so backend logs and profiles keep the names ggml chose. + * + * @param gf Graph to relabel, already built. + */ +inline void graph_unique_names([[maybe_unused]] ggml_cgraph * gf) { +#ifdef GGML_USE_OPENVINO + // Leafs cannot collide: ggml names an unnamed one "leaf_" as it walks the + // graph, and a named one came from the checkpoint. Only results carry a name + // derived from their source, so only nodes are checked. + std::unordered_set seen; + const int n = ggml_graph_n_nodes(gf); + for (int i = 0; i < n; ++i) { + ggml_tensor * t = ggml_graph_node(gf, i); + if (seen.insert(ggml_get_name(t)).second) continue; + ggml_format_name(t, "%s#%d", ggml_get_name(t), i); + seen.insert(ggml_get_name(t)); + } +#endif +} + +/** + * @brief Allocate an arch's weights and tag the buffer as holding weights. + * + * `ggml_backend_alloc_ctx_tensors` leaves the buffer on the default + * `GGML_BACKEND_BUFFER_USAGE_ANY`, which backends read as "this is not weights". + * ggml-openvino takes it literally and classifies every ANY tensor as a KV + * cache, giving it a dynamic sequence dimension; the first translator that asks + * a weight for its static shape then throws `to_shape was called on a dynamic + * shape`. Tagging the buffer is what llama.cpp does with its own weights, and it + * is what lets the OpenVINO frontend fold them in as constants. + * + * @return The weight buffer, or null if the allocation failed (OOM). + */ +inline ggml_backend_buffer_t alloc_weights(ggml_context * ctx, ggml_backend_t backend) { + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (buf) { + ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + } + return buf; +} + /// GPU ordinal for CUDA and SYCL; `VLA_DEVICE` overrides. Junk is rejected, not /// silently read as device 0. inline int backend_device_index() { @@ -153,6 +213,35 @@ inline Backend backend_init(const char * tag, int n_threads) { std::fprintf(stderr, "%s: ggml_backend_metal_init failed; falling back to CPU\n", tag); } } +#elif defined(GGML_USE_OPENVINO) + { + // ggml-openvino translates a graph under 20 nodes literally and anything + // larger through a decoder-only-LLM model builder that infers a KV cache + // and one position input. No vla.cpp graph is that shape and all of them + // are far larger, so raise the bar the literal path is chosen under; + // scripts/patch_ggml_openvino.py is what makes the threshold settable. + // Only a default: an explicit setting wins. ggml caches the value on its + // first OpenVINO entry point, so it has to be set before the init below, + // and call_once because a concurrent model_load would race on the + // environment. + static std::once_flag naive_once; + std::call_once(naive_once, [] { setenv_default("GGML_OPENVINO_NAIVE_GRAPH_SIZE", "1000000"); }); + + // ggml exposes OpenVINO as a single device, so VLA_DEVICE does not apply: + // the target is chosen by name through GGML_OPENVINO_DEVICE (CPU / GPU / + // NPU) and resolved inside ggml, which prints the winner as "OpenVINO: + // using device X" and quietly falls back to CPU when the requested one is + // not enumerated. Echo what was asked for, so the two lines together say + // whether you got the device you wanted. + const char * want = std::getenv("GGML_OPENVINO_DEVICE"); + b.handle = ggml_backend_openvino_init(0); + if (b.handle) { + std::printf("%s: backend = OPENVINO (requested device %s)\n", + tag, (want && *want) ? want : "CPU"); + } else { + std::fprintf(stderr, "%s: ggml_backend_openvino_init failed; falling back to CPU\n", tag); + } + } #endif if (!b.handle) { diff --git a/src/loader.cpp b/src/loader.cpp index 9a67f05..11dc668 100644 --- a/src/loader.cpp +++ b/src/loader.cpp @@ -14,6 +14,8 @@ #include "loader.h" +#include "backend.h" + #include #include #include @@ -134,9 +136,9 @@ bool WeightLoader::upload(ggml_backend_t backend, ggml_backend_buffer_t * out_bu return false; } - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx_, backend); + ggml_backend_buffer_t buf = alloc_weights(ctx_, backend); if (!buf) { - std::fprintf(stderr, "vla(%s): ggml_backend_alloc_ctx_tensors failed (OOM?)\n", arch_); + std::fprintf(stderr, "vla(%s): alloc_weights failed (OOM?)\n", arch_); return false; } *out_buf = buf; diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index a115b5e..a749b20 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -24,6 +24,7 @@ #include "ggml-cuda.h" #endif #include "gguf.h" +#include "backend.h" #include "gguf_reader.h" #include "scratch_ctx.h" @@ -1184,6 +1185,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_build_forward_expand(gf, mm2); if (!vision_scratch.alloc(backend, gf)) { std::fprintf(stderr, "vla(bitvla): gallocr_alloc_graph failed (view %lld)\n", (long long) v); return {}; } ggml_backend_tensor_set(x_in, patches.data(), 0, ggml_nbytes(x_in)); + graph_unique_names(gf); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): vision graph compute failed (view %lld)\n", (long long) v); return {}; @@ -1224,6 +1226,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_build_forward_expand(gf, out); if (!proprio_scratch.alloc(backend, gf)) { std::fprintf(stderr, "vla(bitvla): gallocr failed (proprio)\n"); return {}; } ggml_backend_tensor_set(x_in, state_host.data(), 0, ggml_nbytes(x_in)); + graph_unique_names(gf); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): proprio compute failed\n"); return {}; } ggml_backend_tensor_get(out, proprio_embed_host.data(), 0, (size_t) hidden_l * sizeof(float)); } @@ -1381,6 +1384,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { aids[i] = (int32_t) (seq-2-n_action+i); ggml_backend_tensor_set(action_ids, aids.data(), 0, ggml_nbytes(action_ids)); + graph_unique_names(gf); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): lm prefill compute failed\n"); return {}; } ggml_backend_tensor_get(action_hidden, last_hidden_at_actions.data(), 0, (size_t) n_action * hidden_l * sizeof(float)); } @@ -1430,6 +1434,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_build_forward_expand(gf, y); if (!head_scratch.alloc(backend, gf)) { std::fprintf(stderr, "vla(bitvla): gallocr failed (action_head)\n"); return {}; } ggml_backend_tensor_set(x, last_hidden_at_actions.data(), 0, ggml_nbytes(x)); + graph_unique_names(gf); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): action_head compute failed\n"); return {}; } ggml_backend_tensor_get(y, normalized_actions.data(), 0, (size_t) chunk * action_dim * sizeof(float)); } diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index b1e4d84..d59128d 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -566,6 +566,7 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { if (!preprocess_image_chw(in.images[v], image_size, chw)) { return {}; } ggml_backend_tensor_set(t_px[v], chw.data(), 0, ggml_nbytes(t_px[v])); } + graph_unique_names(vg); if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(evo1): vision graph compute failed (%lld views)\n", (long long) n_views); return {}; @@ -799,6 +800,7 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { { std::vector qm(SEQ, 0.0f); for (int64_t p=0; p Gr00tN1d5ModelArch::predict(const Inputs& in) { for (int64_t v=0; v Gr00tN1d5ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_tproj[s], tpr.data(), 0, ggml_nbytes(gio.t_tproj[s])); } + graph_unique_names(main_graph.graph()); const auto tc0 = std::chrono::steady_clock::now(); const ggml_status status = ggml_backend_graph_compute(backend, main_graph.graph()); const auto tc1 = std::chrono::steady_clock::now(); diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index 52f14fe..d2e49aa 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -374,6 +374,7 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { } if (vok) { ggml_backend_tensor_set(t_patches, patches_all.data(), 0, ggml_nbytes(t_patches)); + graph_unique_names(vgA); if (ggml_backend_graph_compute(backend, vgA) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d6): vision compute A failed\n"); vok = false; @@ -386,6 +387,7 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { shuf_host.data()+(size_t) v*c4*K); ggml_backend_tensor_set(t_shuf, shuf_host.data(), 0, ggml_nbytes(t_shuf)); + graph_unique_names(vgB); if (ggml_backend_graph_compute(backend, vgB) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d6): vision compute B failed\n"); vok = false; @@ -515,6 +517,7 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_tproj[s], tpr.data(), 0, ggml_nbytes(gio.t_tproj[s])); } + graph_unique_names(main_graph.graph()); const auto tc0 = std::chrono::steady_clock::now(); const ggml_status status = ggml_backend_graph_compute(backend, main_graph.graph()); const auto tc1 = std::chrono::steady_clock::now(); diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index ce19844..a64cf7e 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -414,6 +414,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_cos, rope_cos.data(), 0, ggml_nbytes(t_cos)); ggml_backend_tensor_set(t_sin, rope_sin.data(), 0, ggml_nbytes(t_sin)); ggml_backend_tensor_set(t_patches, patches.data(), 0, ggml_nbytes(t_patches)); + graph_unique_names(vg); if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d7): vision compute failed\n"); vok = false; @@ -699,6 +700,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_tproj[s], c_tproj[(size_t) s].data(), 0, ggml_nbytes(t_tproj[s])); } + graph_unique_names(gf); const auto tc0 = std::chrono::steady_clock::now(); const ggml_status st = ggml_backend_graph_compute(backend, gf); const auto tc1 = std::chrono::steady_clock::now(); diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index ed107bf..2c504d3 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -299,6 +299,7 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { normalize_tower(in.images[v],S,DMEAN,DSTD,dbuf); ggml_backend_tensor_set(px_d[v],dbuf.data(),0,ggml_nbytes(px_d[v])); normalize_tower(in.images[v],S,SMEAN,SSTD,sbuf); ggml_backend_tensor_set(px_s[v],sbuf.data(),0,ggml_nbytes(px_s[v])); } + graph_unique_names(vg); if(ggml_backend_graph_compute(backend,vg)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(openvla_oft): vision compute failed\n"); return {}; } ggml_backend_tensor_get(proj,proj_host.data(),0,proj_host.size()*sizeof(float)); stats.ms_vision = std::chrono::duration(clock::now()-tv).count(); @@ -415,6 +416,7 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(act0,z.data(),0,ggml_nbytes(act0)); } + graph_unique_names(gf); if(ggml_backend_graph_compute(backend,gf)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(openvla_oft): main compute failed\n"); return {}; } std::vector na((size_t)action_dim*chunk); ggml_backend_tensor_get(norm_actions,na.data(),0,na.size()*sizeof(float)); diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index c70dcca..824671e 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -510,6 +510,7 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { for (int v=0; v Pi0ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_time[s], tile.data(), 0, ggml_nbytes(t_time[s])); } + graph_unique_names(gf); const auto ti0 = clk::now(); const ggml_status st = ggml_backend_graph_compute(backend, gf); stats.ms_inference = std::chrono::duration(clk::now()-ti0).count(); diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index f16bc56..06ae20c 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -555,6 +555,7 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { for (int v=0; v Pi05ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_time[s], tv.data(), 0, ggml_nbytes(t_time[s])); } + graph_unique_names(gf); const auto ti0 = clk::now(); const ggml_status st = ggml_backend_graph_compute(backend, gf); stats.ms_inference = std::chrono::duration(clk::now()-ti0).count(); diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index 62c062c..0c67029 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -1241,9 +1241,9 @@ SmolVLAModelArch* smolvla_load_impl(ggml_type weight_dtype, cfg.expert_h, cfg.n_suffix); } - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); + m->weight_buf = alloc_weights(m->ctx_weights, m->backend); if (!m->weight_buf) { - std::fprintf(stderr, "vla: ggml_backend_alloc_ctx_tensors (weights) failed\n"); + std::fprintf(stderr, "vla: alloc_weights failed\n"); delete m; return nullptr; } @@ -1551,12 +1551,14 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { break; } ggml_backend_tensor_set(t_px, chw.data(), 0, ggml_nbytes(t_px)); + graph_unique_names(gA); if (ggml_backend_graph_compute(m->backend, gA) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(smolvla): vision compute A failed (view %d)\n", v); vok = false; break; } ggml_backend_tensor_get(post_ln, post_host.data(), 0, ggml_nbytes(post_ln)); pixel_shuffle_hf(post_host.data(), shuf_host.data(), H, grid, s); ggml_backend_tensor_set(t_shuf, shuf_host.data(), 0, ggml_nbytes(t_shuf)); + graph_unique_names(gB); if (ggml_backend_graph_compute(m->backend, gB) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(smolvla): connector compute failed (view %d)\n", v); vok = false; break; } @@ -1687,6 +1689,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_backend_tensor_set(m->in_pos_full, pos_full_host.data(), 0, pos_full_host.size() * sizeof(int32_t)); ggml_backend_tensor_set(m->in_pos_rebased, pos_rebased_host.data(), 0, pos_rebased_host.size() * sizeof(int32_t)); + graph_unique_names(m->gf_cached); const auto t0 = clock::now(); if (ggml_backend_graph_compute(m->backend, m->gf_cached) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla: ggml compute (cached) failed\n"); @@ -1868,8 +1871,13 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { x_t = ggml_add(ctx, x_t, ggml_scale(ctx, v_t, dt)); } + // Graph inputs, not weights: tag them so a backend that reads buffer usage + // (ggml-openvino) does not mistake the default ANY for a KV cache. gallocr + // tags its own arena the same way. ggml_backend_buffer_t compute_buf = ggml_backend_alloc_ctx_tensors(ctx, m->backend); - if (!compute_buf) { + if (compute_buf) { + ggml_backend_buffer_set_usage(compute_buf, GGML_BACKEND_BUFFER_USAGE_COMPUTE); + } else { std::fprintf(stderr, "vla: ggml_backend_alloc_ctx_tensors (compute) failed\n"); ggml_free(ctx); return {}; @@ -1902,6 +1910,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_build_forward_expand(gf_pre, k_cache[i]); ggml_build_forward_expand(gf_pre, v_cache[i]); } + graph_unique_names(gf_pre); const auto t0 = clock::now(); if (ggml_backend_graph_compute(m->backend, gf_pre) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla: ggml prefill compute failed\n"); @@ -1920,6 +1929,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { { ggml_cgraph * gf = ggml_new_graph_custom(ctx, 16384, false); ggml_build_forward_expand(gf, x_t); + graph_unique_names(gf); const auto t0 = clock::now(); if (ggml_backend_graph_compute(m->backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla: ggml compute failed\n"); diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index af00615..e3b1ad2 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -342,6 +342,7 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { normalize_tower(in.images[v],S,DMEAN,DSTD,dbuf); ggml_backend_tensor_set(px_d[v],dbuf.data(),0,ggml_nbytes(px_d[v])); normalize_tower(in.images[v],S,SMEAN,SSTD,sbuf); ggml_backend_tensor_set(px_s[v],sbuf.data(),0,ggml_nbytes(px_s[v])); } + graph_unique_names(vg); if(ggml_backend_graph_compute(backend,vg)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(vla_adapter): vision compute failed\n"); return {}; } ggml_backend_tensor_get(proj,proj_host.data(),0,proj_host.size()*sizeof(float)); stats.ms_vision = std::chrono::duration(clock::now()-tv).count(); @@ -500,6 +501,7 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(cc,cb.data(),0,ggml_nbytes(cc)); ggml_backend_tensor_set(ss,sb.data(),0,ggml_nbytes(ss)); }; fill_cs(cT,sT,chunk); fill_cs(cA,sA,num_tokens+1); fill_cs(cK,sK,NPATCH); + graph_unique_names(gf); if(ggml_backend_graph_compute(backend,gf)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(vla_adapter): main compute failed\n"); return {}; } std::vector na((size_t)action_dim*chunk); ggml_backend_tensor_get(norm_actions,na.data(),0,na.size()*sizeof(float)); diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index 27bc6a0..d297b91 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -426,6 +426,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_pos, c_pos_interp.data(), 0, ggml_nbytes(t_pos)); ggml_backend_tensor_set(t_cos, c_rope_cos.data(), 0, ggml_nbytes(t_cos)); ggml_backend_tensor_set(t_sin, c_rope_sin.data(), 0, ggml_nbytes(t_sin)); + graph_unique_names(vg); if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): vision compute failed\n"); vok = false; @@ -565,6 +566,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_ds[j], ds_pad[j].data(), 0, ggml_nbytes(t_ds[j])); const auto tp0 = std::chrono::steady_clock::now(); + graph_unique_names(lg); if (ggml_backend_graph_compute(backend, lg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): LM compute failed\n"); return {}; } stats.ms_prefill = std::chrono::duration(std::chrono::steady_clock::now()-tp0).count(); if (dump_prefix) { @@ -670,6 +672,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { } const auto td0 = std::chrono::steady_clock::now(); + graph_unique_names(hg); if (ggml_backend_graph_compute(backend, hg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): head compute failed\n"); return {}; } stats.ms_denoise = std::chrono::duration(std::chrono::steady_clock::now()-td0).count(); stats.ms_inference = stats.ms_prefill+stats.ms_denoise; From 428532c5ee9dff27a102ff9507cef377ae84ef7e Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 14:58:44 +0700 Subject: [PATCH 03/24] record the verified GPU run and the Level Zero loader the NPU plugin needs --- docs/backend/ov.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 20f2a37..9433ef6 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -6,13 +6,18 @@ auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. > **Status: builds and runs, no arch completes a prediction yet.** The backend -> comes up, weights fold in, and the vision towers translate and execute. The -> language model and action expert do not: ggml's OpenVINO backend models a -> decoder-only LLM with one position input and an F16 KV cache, and every -> vla.cpp arch has several position inputs and no KV cache. See +> comes up, weights fold in, and the vision towers translate and execute - on +> the CPU plugin and on the GPU plugin alike. The language model and action +> expert do not: ggml's OpenVINO backend models a decoder-only LLM with one +> position input and an F16 KV cache, and every vla.cpp arch has several +> position inputs and no KV cache. See > [What still blocks it](#what-still-blocks-it). The OpenVINO column of the > README support matrix stays `-` until an arch passes end to end. +Checked on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 +iGPU, Ubuntu 24.04, OpenVINO 2026.2.1, against `vrfai/smolvla-libero-gguf` and +`vrfai/pi05-libero-gguf`. + OpenVINO is Intel's inference toolkit; ggml's backend translates a ggml compute graph into an OpenVINO model and hands it to the CPU, GPU or NPU plugin, which compiles and fuses it for the device. Unlike SYCL it needs no separate compiler: @@ -51,6 +56,24 @@ always means the render group has not taken effect yet. Without it, For the GPU compute runtime and NPU driver packages themselves, follow [llama.cpp's OpenVINO notes](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md). +The NPU additionally needs the **Level Zero loader**, which the NPU driver +packages do not pull in: + +```bash +sudo apt-get install -y libze1 # provides libze_loader.so.1 +``` + +`intel-level-zero-npu` ships `libze_intel_npu.so.1`, the *driver*; OpenVINO's NPU +plugin reaches it through the loader and enumerates nothing without one. The +symptom is not an error - the device simply does not appear: + +```text +GGML OpenVINO Backend: device NPU is not available, fallback to CPU +OpenVINO: using device CPU +``` + +`ldconfig -p | grep ze_loader` is the check. + ### 2. OpenVINO runtime + OpenCL headers ```bash From 1f71242aaa36e25025b7cfe385f2c76529525d9d Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 15:05:17 +0700 Subject: [PATCH 04/24] document the two steps the NPU plugin needs and record the NPU run --- docs/backend/ov.md | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 9433ef6..65039c2 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -7,7 +7,7 @@ runtime on the configure line. > **Status: builds and runs, no arch completes a prediction yet.** The backend > comes up, weights fold in, and the vision towers translate and execute - on -> the CPU plugin and on the GPU plugin alike. The language model and action +> the CPU, GPU and NPU plugins alike. The language model and action > expert do not: ggml's OpenVINO backend models a decoder-only LLM with one > position input and an F16 KV cache, and every vla.cpp arch has several > position inputs and no KV cache. See @@ -15,8 +15,11 @@ runtime on the configure line. > README support matrix stays `-` until an arch passes end to end. Checked on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 -iGPU, Ubuntu 24.04, OpenVINO 2026.2.1, against `vrfai/smolvla-libero-gguf` and -`vrfai/pi05-libero-gguf`. +iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, against +`vrfai/smolvla-libero-gguf` and `vrfai/pi05-libero-gguf`. All three devices +behave the same, including the NPU: the naive graph path this build selects is +device-independent, so the NPU never reaches the static prefill/decode path +where it would differ. OpenVINO is Intel's inference toolkit; ggml's backend translates a ggml compute graph into an OpenVINO model and hands it to the CPU, GPU or NPU plugin, which @@ -56,23 +59,36 @@ always means the render group has not taken effect yet. Without it, For the GPU compute runtime and NPU driver packages themselves, follow [llama.cpp's OpenVINO notes](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md). -The NPU additionally needs the **Level Zero loader**, which the NPU driver -packages do not pull in: +The NPU needs two more things that its driver packages do not pull in. Neither +failure is reported as an error - the device simply does not appear, and every +`GGML_OPENVINO_DEVICE=NPU` run lands on the CPU plugin instead: + +```text +GGML OpenVINO Backend: device NPU is not available, fallback to CPU +OpenVINO: using device CPU +``` + +**1. The Level Zero loader.** `intel-level-zero-npu` ships `libze_intel_npu.so.1`, +the *driver*; OpenVINO's NPU plugin only reaches it through the loader. ```bash sudo apt-get install -y libze1 # provides libze_loader.so.1 ``` -`intel-level-zero-npu` ships `libze_intel_npu.so.1`, the *driver*; OpenVINO's NPU -plugin reaches it through the loader and enumerates nothing without one. The -symptom is not an error - the device simply does not appear: +`ldconfig -p | grep ze_loader` is the check. -```text -GGML OpenVINO Backend: device NPU is not available, fallback to CPU -OpenVINO: using device CPU +**2. Point the loader at the NPU driver.** Ubuntu's loader (1.16.1 in noble) +does not discover `libze_intel_npu.so.1` on its own, so installing it is not +enough by itself. Name the driver explicitly: + +```bash +export ZE_ENABLE_ALT_DRIVERS=/lib/x86_64-linux-gnu/libze_intel_npu.so.1 ``` -`ldconfig -p | grep ze_loader` is the check. +With that set the device enumerates as `NPU Intel(R) AI Boost` and the startup +banner reads `OpenVINO: using device NPU`. A loader from Intel's own graphics +repository, version-matched to the NPU driver, should discover it without the +override - untested here. ### 2. OpenVINO runtime + OpenCL headers From 8adb05b084764cb69d2dd1cb3c12098c4acb47bc Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 16:31:22 +0700 Subject: [PATCH 05/24] key rope position inputs per tensor and cache the naive path so smolvla and pi05 run on openvino --- README.md | 22 +-- docs/backend/ov.md | 221 +++++++++++++++++++------- scripts/patch_ggml_openvino.py | 274 +++++++++++++++++++++++++++++++-- src/backend.h | 13 ++ 4 files changed, 448 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index ed0c282..e5c06c0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ A C++ inference engine for **Vision-Language-Action (VLA) models**, built on [`l It runs the open VLA policies - SmolVLA, π0, BitVLA, Evo-1, GR00T N1.5/1.6/1.7 and more - under one runtime, each packaged as a single self-contained GGUF that needs no Python or PyTorch at inference time. The binaries drive robots on **CPU**, **Apple Silicon**, **CUDA** - -from consumer GPUs down to Jetson-class boards - or **Intel GPUs** via SYCL. +from consumer GPUs down to Jetson-class boards - or **Intel GPUs and NPUs** via +SYCL and OpenVINO. [**Learn vla.cpp**](https://fai-modelopt-tech.github.io/learn-vla-cpp/) walks through the engine design and how each policy is implemented on ggml. @@ -27,8 +28,8 @@ from consumer GPUs down to Jetson-class boards - or **Intel GPUs** via SYCL. - CUDA 12.x (optional - required only for CUDA GPU builds) - Intel oneAPI 2025.x + GPU compute runtime (optional - only for Intel GPU builds, see [docs/backend/sycl.md](docs/backend/sycl.md)) -- OpenVINO 2026.x runtime (optional - only for the in-progress OpenVINO backend, - see [docs/backend/ov.md](docs/backend/ov.md)) +- OpenVINO 2026.x runtime (optional - only for Intel CPU/GPU/NPU builds via + OpenVINO, see [docs/backend/ov.md](docs/backend/ov.md)) - `libzmq3-dev`, `cppzmq-dev`, `libprotobuf-dev`, `protobuf-compiler` ```bash @@ -311,15 +312,18 @@ Experimental results on other platforms can be found in Support matrix of models (rows) against platforms (columns). Legend: `Y` = supported (released and benchmarked), `~` = in progress, `-` = planned. -OpenVINO builds and selects its backend today, but no arch completes a -prediction yet - the remaining blocker is upstream in ggml's OpenVINO backend, -written up in [docs/backend/ov.md](docs/backend/ov.md). +OpenVINO covers SmolVLA and π0.5 on Intel CPUs, GPUs and NPUs; on an Arc B390 +iGPU it is 2.9x and 4.4x the native CPU backend, and the NPU beats the CPU +backend on both. The other archs are blocked on ops ggml's OpenVINO backend has +no translator for. Read the known issues in +[docs/backend/ov.md](docs/backend/ov.md) before running it - in particular, do +not set `GGML_OPENVINO_CACHE_DIR`. | Model | CPU (x86-64 / ARM) | CUDA | SYCL (Intel) | Metal | OpenVINO | |---|:--:|:--:|:--:|:--:|:--:| -| [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | - | -| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | - | -| [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | - | +| [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | +| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | ~ | +| [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | | [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | - | | [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | - | | [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | - | diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 65039c2..c90ce32 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -5,21 +5,17 @@ account of how far it currently runs. Like SYCL, OpenVINO is **not** auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. -> **Status: builds and runs, no arch completes a prediction yet.** The backend -> comes up, weights fold in, and the vision towers translate and execute - on -> the CPU, GPU and NPU plugins alike. The language model and action -> expert do not: ggml's OpenVINO backend models a decoder-only LLM with one -> position input and an F16 KV cache, and every vla.cpp arch has several -> position inputs and no KV cache. See -> [What still blocks it](#what-still-blocks-it). The OpenVINO column of the -> README support matrix stays `-` until an arch passes end to end. - -Checked on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 +> **Status: SmolVLA and π0.5 run end to end on CPU, GPU and NPU.** The Arc iGPU +> is 2.9x faster than the native CPU backend on SmolVLA and 4.4x on π0.5. Six fixes +> were needed, four of them inside ggml's OpenVINO backend, which is written +> against llama.cpp's graphs and had never seen a vision tower or an action +> expert - see [What had to change](#what-had-to-change). The other archs are +> blocked on ops the backend has no translator for, listed under +> [What is left](#what-is-left). + +Measured on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, against -`vrfai/smolvla-libero-gguf` and `vrfai/pi05-libero-gguf`. All three devices -behave the same, including the NPU: the naive graph path this build selects is -device-independent, so the NPU never reaches the static prefill/decode path -where it would differ. +`vrfai/smolvla-libero-gguf` and `vrfai/pi05-libero-gguf`. OpenVINO is Intel's inference toolkit; ggml's backend translates a ggml compute graph into an OpenVINO model and hands it to the CPU, GPU or NPU plugin, which @@ -127,9 +123,9 @@ cmake --build build-ov -j$(nproc) binaries: `libopenvino.so` and its TBB live under `/opt/intel`. Configure fails early with a pointer back here if the runtime is not on `CMAKE_PREFIX_PATH`. -`scripts/patch_ggml_openvino.py` runs as the FetchContent patch step, so the -four ggml fixes described in its docstring are applied automatically and -re-applied on a clean reconfigure. There is no manual `git apply`. +`scripts/patch_ggml_openvino.py` runs as the FetchContent patch step, so the six +ggml fixes described in its docstring are applied automatically and re-applied on +a clean reconfigure. There is no manual `git apply`. ## Run @@ -154,14 +150,54 @@ pair tells you whether you got the device you asked for. `VLA_DEVICE` does *not* apply - ggml exposes OpenVINO as a single device and the target is chosen by name. -OpenVINO compiles each graph on first use, which is slow (minutes for a vision -tower). Set `GGML_OPENVINO_CACHE_DIR=` to keep compiled graphs across -restarts, and give any client a receive timeout well above the first request. +OpenVINO compiles each graph on first use, which is slow - a minute or two for a +vision tower on the GPU. Compiled graphs are then cached in-process for the life +of the model, so only the first prediction pays that; give any client a receive +timeout well above the first request. -## What vla.cpp had to change +Do **not** set `GGML_OPENVINO_CACHE_DIR` to carry them across restarts. It +produces silently wrong actions here - see +[Known issues](#known-issues). The backend warns at startup if it is set. -Three of these are ordinary correctness fixes that happen to be invisible on the -other backends: +## Results + +`vla_predict_check` (a test target - add `-DVLA_BUILD_TESTS=ON`), fixed noise, one +camera view, best of 6 iterations (4 for π0.5) after 3 warmups. "CPU backend" is +ggml's own CPU backend on the same 16-core host; the other columns are this build +with `GGML_OPENVINO_DEVICE` set. No `GGML_OPENVINO_CACHE_DIR`, for the reason in +[Known issues](#known-issues). + +| Model | CPU backend | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | +|---|---:|---:|---:|---:| +| SmolVLA (512px) | 1,312 ms | 1,357 ms | **448 ms** (2.9x) | 1,107 ms (1.2x) | +| π0.5 (224px) | 2,775 ms | 4,278 ms | **633 ms** (4.4x) | 931 ms (3.0x) | + +The iGPU is the reason to use this backend. The OpenVINO CPU plugin is at best +parity with ggml's own CPU backend and on π0.5 well behind it, so it is only +worth running to debug a translation. The NPU beats the CPU backend on both +models while drawing far less power, which is the interesting result for a robot. + +Checked against the CPU backend on the same inputs: + +| Run | max abs deviation | RMS | peak action | +|---|---:|---:|---:| +| SmolVLA, OpenVINO CPU | 1.2e-3 | 1.7e-4 | 0.995 | +| SmolVLA, OpenVINO GPU | 1.2e-3 | 1.9e-4 | 0.995 | +| SmolVLA, OpenVINO NPU | 1.6e-2 | 2.1e-3 | 0.995 | +| π0.5, OpenVINO CPU | 8.9e-4 | 8.7e-5 | 0.904 | +| π0.5, OpenVINO GPU | 6.9e-4 | 1.1e-4 | 0.904 | +| π0.5, OpenVINO NPU | 1.6e-3 | 1.9e-4 | 0.904 | + +CPU and GPU sit in the same band as the SYCL backend's numbers - kernel rounding, +plus the F16 K/V conversion the SDPA fix introduces. SmolVLA on the NPU is an +order of magnitude looser because the NPU compile config turns on dynamic +quantization; π0.5 is not, so treat SmolVLA's NPU deviation as a property of that +model on that device rather than of the backend. + +## What had to change + +Two of these are ordinary correctness fixes on the vla.cpp side that happen to be +invisible on the other backends: - **Weight buffers are tagged.** `ggml_backend_alloc_ctx_tensors` leaves a buffer on `GGML_BACKEND_BUFFER_USAGE_ANY`, and ggml-openvino reads ANY as "KV @@ -176,49 +212,120 @@ other backends: duplicates silently collapse into one node and the graph wires up the wrong tensor. `vla::graph_unique_names` relabels duplicates before compute. It compiles to nothing outside an OpenVINO build. + +One is a judgement call about what a weight is: + - **SmolVLA's time tiles moved out of the weight buffer.** They are precomputed once but they are graph inputs, not checkpoint parameters. As weights they became 2-D constants that could not be concatenated with the 4-D activation beside them. -- **`GGML_OPENVINO_NAIVE_GRAPH_SIZE` defaults high.** ggml-openvino translates a - graph under 20 nodes literally and sends anything larger through an LLM model - builder. The literal path is the one that fits a vision tower; the threshold is - raised in `backend_init`, and an explicit setting still wins. -None of it changes what the other backends compute: `vla_predict_check` on a CPU -build of this branch is byte-identical to the same build of `main` for SmolVLA -and π0.5, apart from the `weight_buf` line, which drops by the size of the time -tiles that moved. +And `backend_init` sets one default, the way the SYCL rung already sets +`GGML_SYCL_ENABLE_VMM=0`: -## What still blocks it - -With the above in place, SmolVLA's SigLIP tower translates and runs, and the -prefix/expert graph reaches OpenVINO's shape inference before failing: +- **`GGML_OPENVINO_NAIVE_GRAPH_SIZE` defaults high.** ggml-openvino translates a + graph under 20 nodes literally and sends anything larger through a model + builder that infers a decoder-only LLM. The literal path is the one that fits a + vision tower and an action expert; the threshold is raised in `backend_init`, + and an explicit setting still wins. + +The remaining four are in ggml's OpenVINO backend itself, applied by +`scripts/patch_ggml_openvino.py` at configure time. Its docstring carries the +detail; in short they narrow an llama.cpp-shaped assumption that is stricter than +the ggml contract: + +| Fix | Assumption it relaxes | +|---|---| +| Intel OpenCL platform selection | the first OpenCL platform is Intel's | +| RESHAPE `op_case` guard | a reshape flattening dims 0-2 is the KV-cache flatten | +| SDPA K/V converted with Q | K/V arrive as F16 because the KV cache is | +| **Position inputs keyed per tensor** | **a graph has exactly one position input** | + +The last one is what carries an arch through to a full prediction, and it is the +one worth upstreaming. Every tensor feeding a `GGML_OP_ROPE`'s second input was +renamed to a single parameter called `inp_pos`, and a shared sin/cos table was +built from it. SmolVLA passes three position tensors - prefill, full and rebased - +so they aliased each other and every RoPE took the table built from whichever +won: ```text opset1::Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32]) Argument shapes are inconsistent. ``` -The two operands are RoPE tables of different lengths. `GgmlOvDecoder` maps -*every* tensor feeding a `GGML_OP_ROPE`'s second input to one graph parameter -named `inp_pos`, because an llama.cpp graph has exactly one position input. Every -vla.cpp arch has several - SmolVLA alone passes a prefill, a full and a rebased -position tensor - and they collapse onto each other. - -π0.5 fails on the same node with the same message (`[1,50,1,128]` against -`[1,262,1,128]`), so this is the shared blocker rather than a SmolVLA quirk. - -That is not something vla.cpp can work around from the outside: the fix belongs -in ggml-openvino, which needs to key position inputs per tensor rather than by a -fixed name. The same class of assumption shows up in the KV-cache-shaped dynamic -sequence dimension and in the `compute_op_case` pattern tables, two of which -already needed narrowing (see `scripts/patch_ggml_openvino.py`). - -Separately, several archs use ops the backend has no translator for at all - -`GGML_UNARY_OP_RELU` (every GR00T, Evo-1, VLA-Adapter, OpenVLA-OFT, BitVLA, -VLA-JEPA), `GGML_UNARY_OP_GELU_ERF`, `GGML_OP_NEG`, `GGML_OP_SQR` - and the core -drives a single backend through `gallocr` rather than a scheduler, so there is no -per-op CPU fallback to absorb them. SmolVLA, π0 and π0.5 are the three archs -whose op sets are fully covered today, which is why SmolVLA is the one to retest -first when the position-input handling lands upstream. +When the graph has more than one, each keeps its own name. Nothing is then called +`inp_pos`, the shared-table precompute returns early, and `translate_rope()` +falls back to building sin/cos per op from its own position input - a path that +already existed for mixed RoPE parameters. Graphs with a single position input +are untouched and keep the shared table. + +The fourth fix is about speed rather than correctness: the naive path had no +`graph_key` cache, so it re-converted and re-compiled the whole OpenVINO model on +*every* `ggml_backend_graph_compute`. SmolVLA on the CPU plugin ran at 22.7 s per +prediction before that was fixed and 1.8 s after. + +None of the vla.cpp-side changes alter what the other backends compute: +`vla_predict_check` on a CPU build of this branch is byte-identical to the same +build of `main` for SmolVLA and π0.5, apart from the `weight_buf` line, which +drops by the size of the time tiles that moved. + +## Known issues + +**Do not set `GGML_OPENVINO_CACHE_DIR`.** OpenVINO's on-disk blob cache reloads a +compiled graph that computes the wrong thing. A cold run against a fresh cache +directory is correct; the very next run, reading back the blobs it just wrote, is +not: + +```text +GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_CACHE_DIR=$dir # cold: max |delta| 1.2e-3 +GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_CACHE_DIR=$dir # warm: max |delta| 2.9e0 +``` + +Nothing is logged - the actions are just wrong, which for a policy server is the +worst possible failure mode. `backend_init` warns at startup when the variable is +set. Unverified guess at the cause: the blob key does not capture something that +differs between vla.cpp's several graphs, so one graph gets another's blob. In +practice, pay the compile once per process and leave it unset. + +**SmolVLA's `VLA_TIMING=phase` path is wrong under OpenVINO.** SmolVLA has a +second graph builder used when a caller asks for per-phase timings, and it does +not survive translation - max |delta| 1.9 on every device, with or without the +in-process cache. The default `TimingDetail::NONE` path, which is what +`vla-server` and `vla-cli` use, is correct. On the native CPU backend the two +paths agree exactly, so this is specific to the OpenVINO translation of that +second graph and is not yet diagnosed. π0.5's phase path is unaffected. Per-stage +timings for SmolVLA are therefore omitted from the table above. + +## What is left + +**Op coverage.** The core drives a single backend through `gallocr` rather than a +scheduler, so there is no per-op CPU fallback. An arch that uses an op +ggml-openvino has no translator for cannot run at all: + +| Op | Archs that need it | +|---|---| +| `GGML_UNARY_OP_RELU` | GR00T N1.5/1.6/1.7, Evo-1, VLA-Adapter, OpenVLA-OFT, BitVLA, VLA-JEPA | +| `GGML_UNARY_OP_GELU_ERF` | Evo-1, VLA-Adapter, OpenVLA-OFT, GR00T N1.6, BitVLA | +| `GGML_OP_NEG`, `GGML_OP_SQR` | VLA-JEPA, GR00T N1.7, BitVLA | + +SmolVLA, π0 and π0.5 are the three archs fully covered today. π0 is untested here +only because there was no checkpoint on the machine; its op set matches π0.5's. + +**BitVLA** is a separate case: it pins its ggml graph to the CPU backend by +design and offloads its LM through hand-written CUDA kernels, so an OpenVINO +build leaves it on the CPU regardless. + +**Splitting across devices.** Intel's own +[π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) +puts the vision encoder and language model on the iGPU and the action expert on +the NPU, with the KV cache as the only cross-device handoff. That is a different +toolchain - PyTorch exported to OpenVINO IR as three separate models, no ggml - +so none of it drops into this backend. What does carry over is the shape of the +answer: the two devices are good at different stages, and π0.5 is already within +1.5x of the iGPU on the NPU alone at a fraction of the power. + +vla.cpp cannot make that split today because the core drives one backend for a +whole prediction. It would need a per-*stage* backend rather than a per-op +scheduler - the vision tower, the prefix and the action expert already hand off +through host memory, so the seam is in the right place - but that is an engine +change, not a backend one. diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index 20efeac..8c56759 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -13,14 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Four small fixes to the fetched ggml OpenVINO backend. +"""Six fixes to the fetched ggml OpenVINO backend. ggml-openvino is written against llama.cpp's graphs: one decoder-only transformer, one position input, an F16 KV cache. vla.cpp drives it with vision towers and action experts instead, which is legal ggml but nothing the backend -has seen. Each hunk below is a place where an llama.cpp-shaped assumption is -narrower than the ggml contract. They are what carries a vla.cpp vision tower -through translation; see docs/backend/ov.md for what still does not. +has seen. Five of the hunks below are places where an llama.cpp-shaped +assumption is narrower than the ggml contract; the sixth is a missing cache on +the path those graphs take. Together they are what lets SmolVLA and pi0.5 run +end to end on the CPU, GPU and NPU plugins. Number 4 is the one that matters +most and the one worth upstreaming. + +See docs/backend/ov.md for the measured results and for what is still blocked. 1. ggml-openvino-extra.cpp - pick the *Intel* OpenCL platform. `GGML_OPENVINO_DEVICE=GPU` builds an OpenVINO remote context on an OpenCL @@ -45,7 +49,32 @@ SDPA rejects mixed input types ("Mixed input types are not supported"). Converting K/V too matches the precision the translator already chose. - 4. utils.cpp - make the naive-path graph-size threshold settable. + 4. ggml-decoder.{h,cpp} - stop distinct position inputs aliasing each other. + Every tensor feeding a ROPE's second input is renamed to one graph + parameter called "inp_pos", because an llama.cpp graph has exactly one. + add_rope_sin_cos() then builds a single shared sin/cos table from it. Every + vla.cpp arch has several position tensors -- SmolVLA passes a prefill, a + full and a rebased one -- so they alias each other and every ROPE takes the + table built from whichever won, which fails shape inference: + "Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32]) + Argument shapes are inconsistent." + When the graph has more than one, keep each tensor's own name. Nothing is + then called "inp_pos", so add_rope_sin_cos() returns early and the existing + fallback in translate_rope() builds sin/cos per op from its own position + input. Single-position graphs are untouched and keep the shared table. + This one is what carries an arch through to a full prediction. + + 5. utils.{h,cpp} - cache what the naive path compiles. + The dynamic and static paths keep a `graph_key`-indexed cache of the + decoder and the compiled infer request; the naive path has none, so it + rebuilt the decoder, re-converted the model and called compile_model() on + every single ggml_backend_graph_compute. That is the dominant cost once a + real graph goes through it: SmolVLA on the CPU plugin drops from 22.7 s to + 1.8 s per prediction with the cache in place. A hit rebinds the cached + decoder to the new graph through the existing update_io(), which is how the + dynamic path already handles freshly built tensors. + + 6. utils.cpp - make the naive-path graph-size threshold settable. Graphs under 20 nodes bypass the LLM decoder and translate literally, with static shapes and no KV-cache inference. That literal path is the one that suits a vision tower, but a vision tower is ~450 nodes. The constant @@ -109,6 +138,119 @@ fn = (%s_fn) clGetExtensionFunctionAddressForPlatform(platform, "%s"); """ +NAIVE_COMPUTE_OLD = """enum ggml_status naive_compute(ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config) { + if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { + return GGML_STATUS_SUCCESS; + } + + bool naive = true; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); + auto decoder = std::make_shared(cgraph, model_weights); + auto input_model = std::make_shared(decoder); + auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + ov::serialize(model, "IR_naive.xml"); + } + + std::shared_ptr infer_request; + auto remote_context = ggml_openvino_get_remote_context(); + if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { + // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved + core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::PERFORMANCE)); + } else { + core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); + } + if (remote_context.has_value()) { + infer_request = std::make_shared( + core.compile_model(model, remote_context.value(), config).create_infer_request()); + } else { + infer_request = + std::make_shared(core.compile_model(model, device, config).create_infer_request()); + } + + auto ov_params = model->get_parameters();""" + +NAIVE_COMPUTE_NEW = """enum ggml_status naive_compute(ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config, + std::shared_ptr r_ctx) { + if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { + return GGML_STATUS_SUCCESS; + } + + // vla.cpp: reuse the decoder, the converted model and the compiled infer + // request across calls on the same graph, the way the dynamic and static + // paths already do. Conversion plus compile_model dominates a naive call, so + // without this every graph_compute pays it again. + static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + const graph_key key(cgraph); + + std::shared_ptr entry; + bool cache_hit = false; + if (cache_enabled && r_ctx != nullptr) { + std::lock_guard lock(r_ctx->ctx_mutex); + auto it = r_ctx->naive_cache.find(key); + if (it != r_ctx->naive_cache.end()) { + entry = it->second; + cache_hit = true; + } else { + entry = std::make_shared(); + r_ctx->naive_cache[key] = entry; + } + } else { + entry = std::make_shared(); + } + + // One graph at a time: an ov::InferRequest is not re-entrant, and a hit + // rebinds the decoder to this cgraph. + std::lock_guard entry_lock(entry->mutex); + + bool naive = true; + std::shared_ptr decoder; + std::shared_ptr model; + std::shared_ptr infer_request; + + if (cache_hit && entry->infer_request != nullptr) { + decoder = entry->decoder; + model = entry->model; + infer_request = entry->infer_request; + // Same shapes, new tensors: point the decoder at this call's graph. + decoder->update_io(cgraph); + } else { + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); + decoder = std::make_shared(cgraph, model_weights); + auto input_model = std::make_shared(decoder); + model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + ov::serialize(model, "IR_naive.xml"); + } + + auto remote_context = ggml_openvino_get_remote_context(); + if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { + // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved + core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::PERFORMANCE)); + } else { + core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); + } + if (remote_context.has_value()) { + infer_request = std::make_shared( + core.compile_model(model, remote_context.value(), config).create_infer_request()); + } else { + infer_request = + std::make_shared(core.compile_model(model, device, config).create_infer_request()); + } + + entry->decoder = decoder; + entry->model = model; + entry->infer_request = infer_request; + } + + auto ov_params = model->get_parameters();""" + # file -> [(anchor, replacement), ...]. Every anchor must match exactly once. EDITS = { "ggml/src/ggml-openvino/ggml-openvino-extra.cpp": [ @@ -134,17 +276,6 @@ (USM_LOOKUP % (("clEnqueueMemFillINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemFillINTEL",) * 2)), (USM_LOOKUP % (("clEnqueueMemcpyINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemcpyINTEL",) * 2)), ], - "ggml/src/ggml-openvino/ggml-decoder.cpp": [ - ( - """ } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { - op_case = 3;""", - """ // vla.cpp: case 3 is the KV-cache flatten, whose result is always - // [1, n, 1, 1]. Without the ne[0] test it also swallows the kernel - // reshape ggml_conv_2d emits and rewrites it to the wrong shape. - } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1] && node->ne[0] == 1) { - op_case = 3;""", - ), - ], "ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp": [ ( """ auto q = std::make_shared(q_f32, ov::element::f16);""", @@ -160,7 +291,118 @@ }""", ), ], + "ggml/src/ggml-openvino/ggml-decoder.h": [ + ( + """ std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) { + if (is_inp_pos(tensor, op)) { + return "inp_pos"; + }""", + """ // vla.cpp: only collapse ROPE position inputs onto one "inp_pos" parameter + // when the graph really has one. See scripts/patch_ggml_openvino.py. + bool has_multiple_inp_pos() const; + + std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) { + if (is_inp_pos(tensor, op)) { + return has_multiple_inp_pos() ? std::string(tensor->name) : std::string("inp_pos"); + }""", + ), + ( + " ggml_cgraph * m_cgraph = nullptr;", + " ggml_cgraph * m_cgraph = nullptr;\n" + " mutable int m_multi_inp_pos = -1; // vla.cpp: lazily computed, -1 = unknown", + ), + ], + "ggml/src/ggml-openvino/ggml-decoder.cpp": [ + ( + """ } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { + op_case = 3;""", + """ // vla.cpp: case 3 is the KV-cache flatten, whose result is always + // [1, n, 1, 1]. Without the ne[0] test it also swallows the kernel + // reshape ggml_conv_2d emits and rewrites it to the wrong shape. + } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1] && node->ne[0] == 1) { + op_case = 3;""", + ), + ( + "int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {", + """bool GgmlOvDecoder::has_multiple_inp_pos() const { + if (m_multi_inp_pos < 0) { + std::set seen; + for (int i = 0; i < m_cgraph->n_nodes && seen.size() < 2; i++) { + const ggml_tensor * node = m_cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC && node->src[j] != nullptr; j++) { + if (is_inp_pos(node->src[j], node)) { + seen.insert(node->src[j]); + } + } + } + m_multi_inp_pos = seen.size() > 1 ? 1 : 0; + } + return m_multi_inp_pos == 1; +} + +int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {""", + ), + ], + "ggml/src/ggml-openvino/utils.h": [ + ( + "struct decoder_runtime_ctx {", + """// vla.cpp: what naive_compute() reuses across calls on the same graph. Without +// it that path rebuilt the decoder, re-converted the model and called +// compile_model() on every ggml_backend_graph_compute, which dominated runtime. +struct naive_runtime_ctx { + std::mutex mutex; + std::shared_ptr decoder; + std::shared_ptr model; + std::shared_ptr infer_request; +}; + +struct decoder_runtime_ctx {""", + ), + ( + " std::unordered_map, graph_key_hash> decoder_cache;", + " std::unordered_map, graph_key_hash> decoder_cache;\n" + " std::unordered_map, graph_key_hash> naive_cache;", + ), + ( + """ decoder_cache.clear(); + infer_request_cache.clear();""", + """ decoder_cache.clear(); + naive_cache.clear(); + infer_request_cache.clear();""", + ), + ( + """enum ggml_status naive_compute(struct ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config);""", + """enum ggml_status naive_compute(struct ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config, + std::shared_ptr r_ctx);""", + ), + ], "ggml/src/ggml-openvino/utils.cpp": [ + ( + """ if (!is_model_splitted(cgraph)) { + return naive_compute(cgraph, core, device, config); + }""", + """ if (!is_model_splitted(cgraph)) { + return naive_compute(cgraph, core, device, config, r_ctx); + }""", + ), + ( + """ if (is_naive(cgraph)) { + return naive_compute(cgraph, core, device, config); + }""", + """ if (is_naive(cgraph)) { + return naive_compute(cgraph, core, device, config, r_ctx); + }""", + ), + ( + NAIVE_COMPUTE_OLD, + NAIVE_COMPUTE_NEW, + ), ( """bool is_naive(ggml_cgraph * cgraph) { constexpr int naive_graph_size_threshold = 20;""", diff --git a/src/backend.h b/src/backend.h index 3573495..4dfd5d1 100644 --- a/src/backend.h +++ b/src/backend.h @@ -233,6 +233,19 @@ inline Backend backend_init(const char * tag, int n_threads) { // using device X" and quietly falls back to CPU when the requested one is // not enumerated. Echo what was asked for, so the two lines together say // whether you got the device you wanted. + // OpenVINO's on-disk blob cache reloads a compiled graph that computes + // the wrong thing here: a cold run is correct, and the next run reading + // those blobs back is not, with no error anywhere. Silently wrong + // actions are the worst failure mode a policy server has, so say so + // loudly rather than let it look like a free speedup. + if (const char * cd = std::getenv("GGML_OPENVINO_CACHE_DIR"); cd && *cd) { + std::fprintf(stderr, + "%s: WARNING GGML_OPENVINO_CACHE_DIR is set. Reloading cached blobs has been\n" + "%s: seen to produce silently incorrect actions on the GPU plugin.\n" + "%s: Unset it unless you have verified the outputs. See docs/backend/ov.md.\n", + tag, tag, tag); + } + const char * want = std::getenv("GGML_OPENVINO_DEVICE"); b.handle = ggml_backend_openvino_init(0); if (b.handle) { From 6b343849738b7e5b6c9b72e22fab0858c7edc72d Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 18:25:46 +0700 Subject: [PATCH 06/24] add the missing op translators and pad folded weights to full rank, which unblocks evo-1 --- scripts/patch_ggml_openvino.py | 146 ++++++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 2 deletions(-) diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index 8c56759..b280e8f 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Six fixes to the fetched ggml OpenVINO backend. +"""Nine fixes to the fetched ggml OpenVINO backend. ggml-openvino is written against llama.cpp's graphs: one decoder-only transformer, one position input, an F16 KV cache. vla.cpp drives it with vision @@ -74,7 +74,32 @@ decoder to the new graph through the existing update_io(), which is how the dynamic path already handles freshly built tensors. - 6. utils.cpp - make the naive-path graph-size threshold settable. + 6. openvino/op_table.cpp - add the four missing op translators. + RELU, GELU_ERF, NEG and SQR have no entry in the table, and with no per-op + CPU fallback in the core an arch that uses one cannot run at all. Between + them they block every arch except SmolVLA, pi0 and pi0.5. All four map + onto ov ops directly: Relu, Gelu (whose default is the exact erf form + GELU_ERF asks for), Negative, and -- since no single-input ov op squares -- + Multiply with the input on both sides. + + 7. openvino/utils.cpp - give a folded weight its full rank before slicing. + A ggml tensor that is 2-D folds in as a rank-2 ov constant, which is what a + GEMM operand wants, but process_view_input_new() indexes a viewed tensor at + its full ggml rank, so the slice axis lands outside it: + "Slice (Constant aex.blk.0.attn_in.weight[0]:bf16[2688,896], ...) + Axis 2 out of the tensor rank range [-2, 1]." + Evo-1 hits it by viewing Q, K and V out of one fused attn_in weight. + Left-pad the input with 1s, which is the shape ggml gave it anyway. + + 8. openvino/op/concat.cpp - align input ranks. + The same rank-2 constants reach CONCAT, which unlike the broadcasting + elementwise ops needs both inputs at the graph's rank or the axis falls + outside them. Evo-1 concatenates a CLS weight onto 4-D patch embeddings; + SmolVLA concatenates a precomputed time tile onto a 4-D activation. Padding + here is what let vla.cpp drop an arch-specific workaround that had moved + SmolVLA's time tiles into their own buffer. + + 9. utils.cpp - make the naive-path graph-size threshold settable. Graphs under 20 nodes bypass the LLM decoder and translate literally, with static shapes and no KV-cache inference. That literal path is the one that suits a vision tower, but a vision tower is ~450 nodes. The constant @@ -276,6 +301,123 @@ (USM_LOOKUP % (("clEnqueueMemFillINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemFillINTEL",) * 2)), (USM_LOOKUP % (("clEnqueueMemcpyINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemcpyINTEL",) * 2)), ], + "ggml/src/ggml-openvino/openvino/op/concat.cpp": [ + ( + """#include +#include """, + """#include +#include +#include +#include """, + ), + ("#include ", "#include \n#include "), + ( + """ const auto axis = static_cast(rank - 1 - ggml_dim); + auto res = std::make_shared(OutputVector{input_0, input_1}, axis);""", + """ // vla.cpp: a weight that is 2-D in ggml is folded in as a rank-2 constant, + // because that is what a GEMM operand wants. Concat is the one op where that + // matters: it needs both inputs at the graph's rank, or the axis computed + // below falls outside them. Left-pad the shorter one with 1s, which is the + // shape ggml gave it anyway. + auto align_rank = [rank](ov::Output in) { + const auto & ps = in.get_partial_shape(); + if (ps.rank().is_dynamic() || ps.rank().get_length() >= rank) { + return in; + } + std::vector axes(rank - ps.rank().get_length()); + std::iota(axes.begin(), axes.end(), 0); + auto axes_node = ov::op::v0::Constant::create(ov::element::i64, {axes.size()}, axes); + return ov::Output(std::make_shared(in, axes_node)); + }; + input_0 = align_rank(input_0); + input_1 = align_rank(input_1); + + const auto axis = static_cast(rank - 1 - ggml_dim); + auto res = std::make_shared(OutputVector{input_0, input_1}, axis);""", + ), + ], + "ggml/src/ggml-openvino/openvino/utils.cpp": [ + ("#include ", "#include \n#include "), + ( + "#include ", + "#include \n#include ", + ), + ( + """ size_t view_input_size = context.get_view_input_size(input_index); + if (view_input_size == 0) { + // No view inputs, return the input as is + return input; + } +""", + """ size_t view_input_size = context.get_view_input_size(input_index); + if (view_input_size == 0) { + // No view inputs, return the input as is + return input; + } + + // vla.cpp: a ggml tensor that is 2-D folds in as a rank-2 ov constant, which + // is what a GEMM operand wants, but every slice below indexes the tensor at + // its full ggml rank. Left-pad with 1s so the axes line up. Evo-1 hits this + // by viewing Q/K/V out of one fused attn_in weight. + { + const auto src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, 0); + const auto & in_ps = input.get_partial_shape(); + if (in_ps.rank().is_static() && (size_t) in_ps.rank().get_length() < src_ggml_shape.size()) { + std::vector axes(src_ggml_shape.size() - (size_t) in_ps.rank().get_length()); + std::iota(axes.begin(), axes.end(), 0); + input = std::make_shared( + input, ov::op::v0::Constant::create(ov::element::i64, {axes.size()}, axes)); + } + } +""", + ), + ], + "ggml/src/ggml-openvino/openvino/op_table.cpp": [ + ( + """#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { + +std::unordered_map get_supported_ops() {""", + """#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { + +namespace op { +// vla.cpp: no ov op takes one input and squares it, so pair the input with +// itself rather than route it through Power and a constant exponent. +OutputVector translate_sqr(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, input); + return rename_outputs_with_suffix({res}, context.get_name()); +} +} // namespace op + +std::unordered_map get_supported_ops() {""", + ), + ( + """ {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input },""", + """ {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input }, + // vla.cpp: ov's Gelu defaults to the exact erf formulation, which is what + // GELU_ERF asks for. GGML_UNARY_OP_GELU above is ggml's tanh + // approximation and keeps the mapping it already had. + {"GGML_UNARY_OP_GELU_ERF", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input }, + {"GGML_OP_NEG", op::translate_1to1_match_1_input }, + {"GGML_OP_SQR", op::translate_sqr },""", + ), + ], "ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp": [ ( """ auto q = std::make_shared(q_f32, ov::element::f16);""", From 25154a3d527f7eb9d44e237d6ef5b4f1ccba76a4 Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 19:27:09 +0700 Subject: [PATCH 07/24] map ggml's neg to the unary op table key so vla-jepa translates --- scripts/patch_ggml_openvino.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index b280e8f..5710351 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -75,7 +75,7 @@ dynamic path already handles freshly built tensors. 6. openvino/op_table.cpp - add the four missing op translators. - RELU, GELU_ERF, NEG and SQR have no entry in the table, and with no per-op + RELU, GELU_ERF, NEG (a unary op) and SQR have no entry in the table, and with no per-op CPU fallback in the core an arch that uses one cannot run at all. Between them they block every arch except SmolVLA, pi0 and pi0.5. All four map onto ov ops directly: Relu, Gelu (whose default is the exact erf form @@ -414,7 +414,7 @@ // approximation and keeps the mapping it already had. {"GGML_UNARY_OP_GELU_ERF", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input }, - {"GGML_OP_NEG", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input }, {"GGML_OP_SQR", op::translate_sqr },""", ), ], From 551ebe7aa8add7415028e7f6eef8faf9eb71698f Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 19:36:06 +0700 Subject: [PATCH 08/24] record the five verified architectures and the three npu failure modes --- README.md | 15 ++- docs/backend/ov.md | 276 ++++++++++++++++++++++++++------------------- 2 files changed, 164 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index e5c06c0..28868d8 100644 --- a/README.md +++ b/README.md @@ -312,10 +312,9 @@ Experimental results on other platforms can be found in Support matrix of models (rows) against platforms (columns). Legend: `Y` = supported (released and benchmarked), `~` = in progress, `-` = planned. -OpenVINO covers SmolVLA and π0.5 on Intel CPUs, GPUs and NPUs; on an Arc B390 -iGPU it is 2.9x and 4.4x the native CPU backend, and the NPU beats the CPU -backend on both. The other archs are blocked on ops ggml's OpenVINO backend has -no translator for. Read the known issues in +OpenVINO covers SmolVLA, π0.5, Evo-1, VLA-Adapter and VLA-JEPA on Intel CPUs, +GPUs and NPUs; on an Arc B390 iGPU it is 3.1x to 8.2x the native CPU backend. +The NPU takes two of the five. Read the known issues in [docs/backend/ov.md](docs/backend/ov.md) before running it - in particular, do not set `GGML_OPENVINO_CACHE_DIR`. @@ -328,10 +327,10 @@ not set `GGML_OPENVINO_CACHE_DIR`. | [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | - | | [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | - | | [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | -| [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | - | -| [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | - | -| [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | - | -| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | - | +| [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | +| [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | +| [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | ~ | +| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | --- diff --git a/docs/backend/ov.md b/docs/backend/ov.md index c90ce32..5d8aed9 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -5,17 +5,17 @@ account of how far it currently runs. Like SYCL, OpenVINO is **not** auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. -> **Status: SmolVLA and π0.5 run end to end on CPU, GPU and NPU.** The Arc iGPU -> is 2.9x faster than the native CPU backend on SmolVLA and 4.4x on π0.5. Six fixes -> were needed, four of them inside ggml's OpenVINO backend, which is written +> **Status: five architectures run end to end.** SmolVLA, π0.5, Evo-1, +> VLA-Adapter and VLA-JEPA all produce actions matching the CPU backend, on the +> CPU and GPU plugins; the NPU takes two of the five. On the Arc B390 iGPU the +> speedup over the native CPU backend runs from 3.1x to 8.2x. Nine fixes were +> needed, all but two of them inside ggml's OpenVINO backend, which is written > against llama.cpp's graphs and had never seen a vision tower or an action -> expert - see [What had to change](#what-had-to-change). The other archs are -> blocked on ops the backend has no translator for, listed under -> [What is left](#what-is-left). +> expert - see [What had to change](#what-had-to-change). Measured on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 -iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, against -`vrfai/smolvla-libero-gguf` and `vrfai/pi05-libero-gguf`. +iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10331` +(the tag `CMakeLists.txt` pins), on the checkpoints under `vrfai/` on the Hub. OpenVINO is Intel's inference toolkit; ggml's backend translates a ggml compute graph into an OpenVINO model and hands it to the CPU, GPU or NPU plugin, which @@ -161,92 +161,107 @@ produces silently wrong actions here - see ## Results -`vla_predict_check` (a test target - add `-DVLA_BUILD_TESTS=ON`), fixed noise, one -camera view, best of 6 iterations (4 for π0.5) after 3 warmups. "CPU backend" is -ggml's own CPU backend on the same 16-core host; the other columns are this build -with `GGML_OPENVINO_DEVICE` set. No `GGML_OPENVINO_CACHE_DIR`, for the reason in -[Known issues](#known-issues). - -| Model | CPU backend | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | -|---|---:|---:|---:|---:| -| SmolVLA (512px) | 1,312 ms | 1,357 ms | **448 ms** (2.9x) | 1,107 ms (1.2x) | -| π0.5 (224px) | 2,775 ms | 4,278 ms | **633 ms** (4.4x) | 931 ms (3.0x) | - -The iGPU is the reason to use this backend. The OpenVINO CPU plugin is at best -parity with ggml's own CPU backend and on π0.5 well behind it, so it is only -worth running to debug a translation. The NPU beats the CPU backend on both -models while drawing far less power, which is the interesting result for a robot. - -Checked against the CPU backend on the same inputs: - -| Run | max abs deviation | RMS | peak action | -|---|---:|---:|---:| -| SmolVLA, OpenVINO CPU | 1.2e-3 | 1.7e-4 | 0.995 | -| SmolVLA, OpenVINO GPU | 1.2e-3 | 1.9e-4 | 0.995 | -| SmolVLA, OpenVINO NPU | 1.6e-2 | 2.1e-3 | 0.995 | -| π0.5, OpenVINO CPU | 8.9e-4 | 8.7e-5 | 0.904 | -| π0.5, OpenVINO GPU | 6.9e-4 | 1.1e-4 | 0.904 | -| π0.5, OpenVINO NPU | 1.6e-3 | 1.9e-4 | 0.904 | - -CPU and GPU sit in the same band as the SYCL backend's numbers - kernel rounding, -plus the F16 K/V conversion the SDPA fix introduces. SmolVLA on the NPU is an -order of magnitude looser because the NPU compile config turns on dynamic -quantization; π0.5 is not, so treat SmolVLA's NPU deviation as a property of that -model on that device rather than of the backend. +`vla_predict_check` (a test target - add `-DVLA_BUILD_TESTS=ON`), fixed noise, +one camera view, best of 4-6 iterations after 3 warmups. "CPU backend" is ggml's +own CPU backend on the same 16-core host. No `GGML_OPENVINO_CACHE_DIR`, for the +reason in [Known issues](#known-issues). + +| Model | input | CPU backend | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | +|---|---|---:|---:|---:|---:| +| SmolVLA | 512 | 1,364 ms | 1,340 ms | **446 ms** (3.1x) | 1,162 ms | +| π0.5 | 224 | 2,802 ms | 4,285 ms | **641 ms** (4.4x) | 916 ms | +| Evo-1 | 448 | 3,114 ms | 4,523 ms | **574 ms** (5.4x) | not supported | +| VLA-Adapter | 224 | 1,228 ms | 1,603 ms | **162 ms** (7.6x) | not supported | +| VLA-JEPA | 256 | 1,046 ms | 1,265 ms | **128 ms** (8.2x) | not supported | + +The iGPU is the reason to use this backend, and it pays off most where the model +is most vision-heavy. The OpenVINO CPU plugin is at best parity with ggml's own +CPU backend and often well behind it, so it is only worth running to debug a +translation. The NPU beats the CPU backend on the two models it accepts while +drawing far less power, which is the interesting result for a robot - see the +NPU limits under [Known issues](#known-issues). + +Actions checked against the CPU backend on identical inputs. Both sides are +deterministic run to run, so these are exact, not sampled: + +| Model | device | max abs deviation | RMS | peak action | +|---|---|---:|---:|---:| +| SmolVLA | CPU | 1.2e-3 | 1.7e-4 | 0.995 | +| SmolVLA | GPU | 1.2e-3 | 1.9e-4 | 0.995 | +| SmolVLA | NPU | 1.6e-2 | 2.1e-3 | 0.995 | +| π0.5 | CPU | 8.9e-4 | 8.7e-5 | 0.904 | +| π0.5 | GPU | 6.9e-4 | 1.1e-4 | 0.904 | +| π0.5 | NPU | 1.6e-3 | 1.9e-4 | 0.904 | +| Evo-1 | CPU | 2.7e-3 | 3.8e-4 | 0.899 | +| Evo-1 | GPU | 2.9e-3 | 4.2e-4 | 0.899 | +| VLA-Adapter | CPU | 2.1e-3 | 7.1e-4 | 0.662 | +| VLA-Adapter | GPU | 2.9e-3 | 1.1e-3 | 0.662 | +| VLA-JEPA | CPU | 1.2e-2 | 4.1e-3 | 1.145 | +| VLA-JEPA | GPU | 9.1e-3 | 4.5e-3 | 1.145 | + +Most of these sit in the same band as the SYCL backend's numbers - kernel +rounding, plus the F16 K/V conversion the SDPA fix introduces. + +Two rows are looser and worth naming rather than burying. SmolVLA on the NPU is +an order of magnitude off because the NPU compile config turns on dynamic +quantization; π0.5 on the same device is not, so treat it as a property of that +model on that device. **VLA-JEPA is the loosest CPU/GPU result at ~1% relative, +and it is not diagnosed** - the error is spread evenly across the action vector +rather than sitting in one element, and VLA-JEPA does not use flash attention, so +the SDPA conversion is not the cause. Verify it against your own policy before +trusting VLA-JEPA on this backend. ## What had to change -Two of these are ordinary correctness fixes on the vla.cpp side that happen to be -invisible on the other backends: +Two fixes on the vla.cpp side. Both are ordinary correctness fixes that happen to +be invisible on the other backends: -- **Weight buffers are tagged.** `ggml_backend_alloc_ctx_tensors` leaves a - buffer on `GGML_BACKEND_BUFFER_USAGE_ANY`, and ggml-openvino reads ANY as "KV - cache", giving every weight a dynamic sequence dimension. `vla::alloc_weights` - in [`src/backend.h`](../../src/backend.h) tags it `..._WEIGHTS`, which is what +- **Weight buffers are tagged.** `ggml_backend_alloc_ctx_tensors` leaves a buffer + on `GGML_BACKEND_BUFFER_USAGE_ANY`, and ggml-openvino reads ANY as "KV cache", + giving every weight a dynamic sequence dimension. `vla::alloc_weights` in + [`src/backend.h`](../../src/backend.h) tags it `..._WEIGHTS`, which is what llama.cpp does with its own weights and what lets the frontend fold them in as - constants. + constants. Since 0.3.0 every arch allocates through `vla::WeightLoader`, so + this is one call site in [`src/loader.cpp`](../../src/loader.cpp). - **Graph tensors get unique names.** ggml derives a result's name from its source, so `ggml_reshape_2d` of an unnamed tensor is called `" (reshaped)"` - and a graph whose intermediates were never named ends up with many tensors sharing one name. ggml-openvino keys its translation map on those names, so - duplicates silently collapse into one node and the graph wires up the wrong - tensor. `vla::graph_unique_names` relabels duplicates before compute. It - compiles to nothing outside an OpenVINO build. - -One is a judgement call about what a weight is: - -- **SmolVLA's time tiles moved out of the weight buffer.** They are precomputed - once but they are graph inputs, not checkpoint parameters. As weights they - became 2-D constants that could not be concatenated with the 4-D activation - beside them. - -And `backend_init` sets one default, the way the SYCL rung already sets -`GGML_SYCL_ENABLE_VMM=0`: - -- **`GGML_OPENVINO_NAIVE_GRAPH_SIZE` defaults high.** ggml-openvino translates a - graph under 20 nodes literally and sends anything larger through a model - builder that infers a decoder-only LLM. The literal path is the one that fits a - vision tower and an action expert; the threshold is raised in `backend_init`, - and an explicit setting still wins. - -The remaining four are in ggml's OpenVINO backend itself, applied by + duplicates silently collapse into one node and the graph wires the wrong tensor + into the next op. `vla::graph_unique_names` relabels duplicates before compute, + at each of the 29 `ggml_backend_graph_compute` call sites. It compiles to + nothing outside an OpenVINO build. + +`backend_init` also sets one default, the way the SYCL rung already sets +`GGML_SYCL_ENABLE_VMM=0`: **`GGML_OPENVINO_NAIVE_GRAPH_SIZE` defaults high.** +ggml-openvino translates a graph under 20 nodes literally and sends anything +larger through a model builder that assumes a decoder-only LLM. The literal path +is the one that fits a vision tower and an action expert. An explicit setting +still wins. + +The other seven are in ggml's OpenVINO backend itself, applied by `scripts/patch_ggml_openvino.py` at configure time. Its docstring carries the -detail; in short they narrow an llama.cpp-shaped assumption that is stricter than -the ggml contract: +detail; in short each narrows an llama.cpp-shaped assumption that is stricter +than the ggml contract, or fills a gap: -| Fix | Assumption it relaxes | +| Fix | What it addresses | |---|---| -| Intel OpenCL platform selection | the first OpenCL platform is Intel's | -| RESHAPE `op_case` guard | a reshape flattening dims 0-2 is the KV-cache flatten | -| SDPA K/V converted with Q | K/V arrive as F16 because the KV cache is | -| **Position inputs keyed per tensor** | **a graph has exactly one position input** | - -The last one is what carries an arch through to a full prediction, and it is the -one worth upstreaming. Every tensor feeding a `GGML_OP_ROPE`'s second input was -renamed to a single parameter called `inp_pos`, and a shared sin/cos table was -built from it. SmolVLA passes three position tensors - prefill, full and rebased - -so they aliased each other and every RoPE took the table built from whichever -won: +| Intel OpenCL platform selection | assumes the first OpenCL platform is Intel's | +| RESHAPE `op_case` guard | assumes a reshape flattening dims 0-2 is the KV-cache flatten | +| SDPA K/V converted with Q | assumes K/V arrive as F16 because the KV cache is | +| **Position inputs keyed per tensor** | **assumes a graph has exactly one position input** | +| Folded weights padded to full rank | a 2-D weight becomes a rank-2 constant, but views index it at ggml rank | +| CONCAT input ranks aligned | same rank-2 constants, and concat cannot broadcast rank | +| Missing op translators | RELU, GELU_ERF, NEG, SQR had no table entry | +| Naive-path graph cache | that path re-compiled the whole model on every graph_compute | + +Three are worth expanding. + +**Position inputs** is what carries an arch through to a full prediction. Every +tensor feeding a `GGML_OP_ROPE`'s second input was renamed to a single parameter +called `inp_pos`, and one shared sin/cos table was built from it. SmolVLA passes +three position tensors - prefill, full and rebased - so they aliased each other +and every RoPE took the table built from whichever won: ```text opset1::Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32]) @@ -256,18 +271,25 @@ Argument shapes are inconsistent. When the graph has more than one, each keeps its own name. Nothing is then called `inp_pos`, the shared-table precompute returns early, and `translate_rope()` falls back to building sin/cos per op from its own position input - a path that -already existed for mixed RoPE parameters. Graphs with a single position input -are untouched and keep the shared table. - -The fourth fix is about speed rather than correctness: the naive path had no -`graph_key` cache, so it re-converted and re-compiled the whole OpenVINO model on -*every* `ggml_backend_graph_compute`. SmolVLA on the CPU plugin ran at 22.7 s per -prediction before that was fixed and 1.8 s after. +already existed for mixed RoPE parameters. Single-position graphs are untouched. + +**Rank padding** is the other structural one. A ggml tensor that is 2-D folds in +as a rank-2 constant, which is what a GEMM operand wants, but the graph indexes +it at full ggml rank. Evo-1 views Q, K and V out of one fused `attn_in` weight +and got `Axis 2 out of the tensor rank range [-2, 1]`. Padding in +`process_view_input_new` fixed that class generally, and padding in the concat +translator let vla.cpp **delete** an arch-specific workaround that had moved +SmolVLA's time tiles into their own buffer. + +**The naive-path cache** is about speed, not correctness. The dynamic and static +paths keep a `graph_key`-indexed cache; the naive path had none, so it rebuilt +the decoder, re-converted the model and called `compile_model()` on every single +`ggml_backend_graph_compute`. SmolVLA on the CPU plugin ran at 22.7 s per +prediction before, 1.4 s after. None of the vla.cpp-side changes alter what the other backends compute: `vla_predict_check` on a CPU build of this branch is byte-identical to the same -build of `main` for SmolVLA and π0.5, apart from the `weight_buf` line, which -drops by the size of the time tiles that moved. +build of the base commit, for every model tested. ## Known issues @@ -281,48 +303,64 @@ GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_CACHE_DIR=$dir # cold: max |delta| 1.2e GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_CACHE_DIR=$dir # warm: max |delta| 2.9e0 ``` -Nothing is logged - the actions are just wrong, which for a policy server is the -worst possible failure mode. `backend_init` warns at startup when the variable is -set. Unverified guess at the cause: the blob key does not capture something that -differs between vla.cpp's several graphs, so one graph gets another's blob. In -practice, pay the compile once per process and leave it unset. +Nothing is logged - the actions are simply wrong, which for a policy server is +the worst possible failure mode. `backend_init` warns at startup when the +variable is set. Unverified guess at the cause: the blob key does not capture +something that differs between vla.cpp's several graphs, so one graph gets +another's blob. In practice, pay the compile once per process and leave it unset. + +**The NPU takes two of the five archs, and fails three different ways.** SmolVLA +and π0.5 run. The others do not: + +| Model | NPU outcome | +|---|---| +| Evo-1 | compiler rejects: `Input channels '1025' is not aligned by '16'` | +| VLA-Adapter | compiler rejects: `Input channels '261' is not aligned by '16'` | +| VLA-JEPA | compiles and runs, returns all `NaN` | + +The two rejections are Intel's NPU compiler, not vla.cpp: 1025 is Evo-1's 1024 +patches plus a CLS token, 261 is VLA-Adapter's 256 plus 5, and neither is a +multiple of 16. SmolVLA and π0.5 happen to have 16-aligned sequence lengths. The +VLA-JEPA NaN is a third failure mode and is not diagnosed. Note that a +partially-failing NPU run still reports a wall-clock time, so do not read a +latency number off a run whose actions did not come out. **SmolVLA's `VLA_TIMING=phase` path is wrong under OpenVINO.** SmolVLA has a second graph builder used when a caller asks for per-phase timings, and it does not survive translation - max |delta| 1.9 on every device, with or without the in-process cache. The default `TimingDetail::NONE` path, which is what -`vla-server` and `vla-cli` use, is correct. On the native CPU backend the two -paths agree exactly, so this is specific to the OpenVINO translation of that -second graph and is not yet diagnosed. π0.5's phase path is unaffected. Per-stage -timings for SmolVLA are therefore omitted from the table above. +`vla-server` and `vla-cli` use, is correct, and on the native CPU backend the two +paths agree exactly. Evo-1 and π0.5 are unaffected on the same path, so this is +specific to SmolVLA's second graph. One hypothesis - that the split-graph guard +sends it down the LLM path - was tested and is wrong: forcing the naive path on +split graphs returns zeros. Per-stage timings for SmolVLA are therefore omitted +from the tables above. ## What is left -**Op coverage.** The core drives a single backend through `gallocr` rather than a -scheduler, so there is no per-op CPU fallback. An arch that uses an op -ggml-openvino has no translator for cannot run at all: - -| Op | Archs that need it | -|---|---| -| `GGML_UNARY_OP_RELU` | GR00T N1.5/1.6/1.7, Evo-1, VLA-Adapter, OpenVLA-OFT, BitVLA, VLA-JEPA | -| `GGML_UNARY_OP_GELU_ERF` | Evo-1, VLA-Adapter, OpenVLA-OFT, GR00T N1.6, BitVLA | -| `GGML_OP_NEG`, `GGML_OP_SQR` | VLA-JEPA, GR00T N1.7, BitVLA | - -SmolVLA, π0 and π0.5 are the three archs fully covered today. π0 is untested here -only because there was no checkpoint on the machine; its op set matches π0.5's. - -**BitVLA** is a separate case: it pins its ggml graph to the CPU backend by -design and offloads its LM through hand-written CUDA kernels, so an OpenVINO -build leaves it on the CPU regardless. +**Op coverage is no longer the blocker.** With RELU, GELU_ERF, NEG and SQR added +to the table, every ggml op the eleven in-tree archs build is translatable. The +one exception is `ggml_map_custom1`, used only by BitVLA - and BitVLA pins its +ggml graph to the CPU backend by design and offloads its LM through hand-written +CUDA kernels, so an OpenVINO build leaves it on the CPU regardless. `GGML_OP_SQR` +is therefore **untested**: only BitVLA emits it, and BitVLA never reaches this +backend. It is in the table because it is a real gap in ggml-openvino, not +because anything here exercises it. + +**Untested archs.** π0 and OpenVLA-OFT are untested here for want of a local +checkpoint, not because anything is known to block them; both use op sets already +covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). The +GR00T family is being brought up separately. Treat any untested arch as `-` in +the README matrix until it has actually produced actions. **Splitting across devices.** Intel's own [π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) puts the vision encoder and language model on the iGPU and the action expert on the NPU, with the KV cache as the only cross-device handoff. That is a different toolchain - PyTorch exported to OpenVINO IR as three separate models, no ggml - -so none of it drops into this backend. What does carry over is the shape of the -answer: the two devices are good at different stages, and π0.5 is already within -1.5x of the iGPU on the NPU alone at a fraction of the power. +so none of it drops into this backend. What carries over is the shape of the +answer: the two devices suit different stages, and π0.5 on the NPU alone is +already within 1.5x of the iGPU at a fraction of the power. vla.cpp cannot make that split today because the core drives one backend for a whole prediction. It would need a per-*stage* backend rather than a per-op From f6f0e2cefba6066e2ee100e7d71c28ac0bf14e5c Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 19:52:19 +0700 Subject: [PATCH 09/24] honour interleaved mrope sections and mode, and record gr00t n1.7 as numerically wrong --- docs/backend/ov.md | 17 ++++++++-- scripts/patch_ggml_openvino.py | 61 ++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 5d8aed9..cfa0d45 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -347,11 +347,22 @@ is therefore **untested**: only BitVLA emits it, and BitVLA never reaches this backend. It is in the table because it is a real gap in ggml-openvino, not because anything here exercises it. +**GR00T N1.7 translates but computes the wrong answer.** It runs to completion +and returns a full action chunk, so nothing errors, but 86% of the 5,280 values +are off by more than 1e-2 and the worst is 1.48 against a peak of 0.948. The +result is deterministic and identical on the CPU and GPU plugins, so this is a +translation bug rather than a device or precision effect. It is **not** the mrope +handling: fixes 9 and 10 in `scripts/patch_ggml_openvino.py` both target that +path and neither moved the number by a digit. VLA-JEPA shares the same Qwen3-VL +vision tower and the same interleaved mrope and is only ~1% off, which points at +GR00T N1.7's own action expert rather than anything shared. Not yet diagnosed; +the arch stays `-` in the README matrix. + **Untested archs.** π0 and OpenVLA-OFT are untested here for want of a local checkpoint, not because anything is known to block them; both use op sets already -covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). The -GR00T family is being brought up separately. Treat any untested arch as `-` in -the README matrix until it has actually produced actions. +covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). GR00T +N1.5 and N1.6 are still being fetched. Treat any untested arch as `-` in the +README matrix until it has actually produced actions. **Splitting across devices.** Intel's own [π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index 5710351..3821325 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Nine fixes to the fetched ggml OpenVINO backend. +"""Eleven fixes to the fetched ggml OpenVINO backend. ggml-openvino is written against llama.cpp's graphs: one decoder-only transformer, one position input, an F16 KV cache. vla.cpp drives it with vision @@ -99,7 +99,26 @@ here is what let vla.cpp drop an arch-specific workaround that had moved SmolVLA's time tiles into their own buffer. - 9. utils.cpp - make the naive-path graph-size threshold settable. + 9. openvino/utils.cpp - bound the interleaved-mrope sector cycle by sections. + ggml's IMROPE cycles t/h/w by sector % 3, but only while the sector is + inside 3 * sections[k]; past that it falls through to the fourth position + stream (ggml_rope_cache_init in ggml/src/ggml-cpu/ops.cpp). The translator + cycled unconditionally, so with sections {24,20,20,0} and n_dims 128 + sectors 61 and 62 took h and w instead of e. NOTE: this matches the ggml + reference but had no measurable effect on any arch tested here, because the + fourth stream happens to carry the same positions as the first. Kept + because it removes a real divergence from the reference, not because a + measurement demanded it. + + 10. openvino/translate_session.cpp - pass the mode to the shared sin/cos table. + add_rope_sin_cos() called make_sin_cos() without the imrope flag, so a graph + with a single position input and interleaved mrope got a table built with + the plain-rope layout - silently wrong, not an error. Same caveat as 9: every + mrope arch here has several position inputs, so the shared precompute is + skipped and this path is untested. It is strictly closer to the reference + than what it replaces. + + 11. utils.cpp - make the naive-path graph-size threshold settable. Graphs under 20 nodes bypass the LLM decoder and translate literally, with static shapes and no KV-cache inference. That literal path is the one that suits a vision tower, but a vision tower is ~450 nodes. The constant @@ -338,6 +357,34 @@ ], "ggml/src/ggml-openvino/openvino/utils.cpp": [ ("#include ", "#include \n#include "), + ( + """ std::vector gather_indices(n_dims_half); + for (size_t j = 0; j < n_dims_half; j++) { + gather_indices[j] = j % 3; + factor[j] = std::pow(theta_scale, j); + }""", + """ // vla.cpp: ggml's interleaved mrope cycles t/h/w by sector % 3, but only + // while the sector is still inside 3 * sections[k]; past that it falls + // through to the fourth position stream. Ignoring the bound sends the + // tail sectors to the wrong stream -- with sections {24,20,20,0} and + // n_dims 128, sectors 61 and 62 take h and w instead of e. See + // ggml_rope_cache_init in ggml/src/ggml-cpu/ops.cpp. + const int32_t * sections = rope_params + 11; + std::vector gather_indices(n_dims_half); + for (size_t j = 0; j < n_dims_half; j++) { + const int sector = (int) j; + int64_t stream = 3; + if (sector % 3 == 1 && sector < 3 * sections[1]) { + stream = 1; + } else if (sector % 3 == 2 && sector < 3 * sections[2]) { + stream = 2; + } else if (sector % 3 == 0 && sector < 3 * sections[0]) { + stream = 0; + } + gather_indices[j] = stream; + factor[j] = std::pow(theta_scale, j); + }""", + ), ( "#include ", "#include \n#include ", @@ -557,6 +604,16 @@ }();""", ), ], + "ggml/src/ggml-openvino/openvino/translate_session.cpp": [ + ( + " auto sin_cos = make_sin_cos(rope_params, inp_pos, rope_freqs_weight);", + """ // vla.cpp: rope_params[2] is the mode. The shared precompute never looked at + // it, so a graph with one position input and interleaved mrope got a table + // built with the plain-rope layout -- silently wrong actions, not an error. + const bool imrope = rope_params[2] == GGML_ROPE_TYPE_IMROPE; + auto sin_cos = make_sin_cos(rope_params, inp_pos, rope_freqs_weight, imrope, false);""", + ), + ], } From 07b5d73194984adffaa77cbb3f6781c6685cb85f Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 20:24:36 +0700 Subject: [PATCH 10/24] grade the archs by how far they drift from the cpu backend and add gr00t n1.5 --- README.md | 11 ++++--- docs/backend/ov.md | 82 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 28868d8..5e4adff 100644 --- a/README.md +++ b/README.md @@ -312,9 +312,10 @@ Experimental results on other platforms can be found in Support matrix of models (rows) against platforms (columns). Legend: `Y` = supported (released and benchmarked), `~` = in progress, `-` = planned. -OpenVINO covers SmolVLA, π0.5, Evo-1, VLA-Adapter and VLA-JEPA on Intel CPUs, -GPUs and NPUs; on an Arc B390 iGPU it is 3.1x to 8.2x the native CPU backend. -The NPU takes two of the five. Read the known issues in +OpenVINO runs SmolVLA, π0.5, Evo-1 and VLA-Adapter on Intel CPUs, GPUs and NPUs, +matching the CPU backend to 3e-3; on an Arc B390 iGPU that is 3.1x to 8.2x the +native CPU backend. VLA-JEPA and GR00T N1.5 run but drift further than that and +are marked in progress. Read the known issues in [docs/backend/ov.md](docs/backend/ov.md) before running it - in particular, do not set `GGML_OPENVINO_CACHE_DIR`. @@ -323,14 +324,14 @@ not set `GGML_OPENVINO_CACHE_DIR`. | [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | | [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | ~ | | [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | -| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | - | +| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | ~ | | [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | - | | [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | - | | [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | | [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | | [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | | [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | ~ | -| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | +| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | ~ | --- diff --git a/docs/backend/ov.md b/docs/backend/ov.md index cfa0d45..50c29d7 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -5,13 +5,16 @@ account of how far it currently runs. Like SYCL, OpenVINO is **not** auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. -> **Status: five architectures run end to end.** SmolVLA, π0.5, Evo-1, -> VLA-Adapter and VLA-JEPA all produce actions matching the CPU backend, on the -> CPU and GPU plugins; the NPU takes two of the five. On the Arc B390 iGPU the -> speedup over the native CPU backend runs from 3.1x to 8.2x. Nine fixes were -> needed, all but two of them inside ggml's OpenVINO backend, which is written -> against llama.cpp's graphs and had never seen a vision tower or an action -> expert - see [What had to change](#what-had-to-change). +> **Status: four architectures match the CPU backend, two more run but drift, +> one is wrong.** SmolVLA, π0.5, Evo-1 and VLA-Adapter agree with the CPU backend +> to 3e-3 or better - the same bar the SYCL backend is held to. VLA-JEPA and +> GR00T N1.5 run to completion but deviate 4-20x more than that, which is not yet +> explained and is enough to matter for control. GR00T N1.7 translates and returns +> plausible-looking actions that are simply wrong. On the Arc B390 iGPU the +> speedup over the native CPU backend runs from 3.1x to 8.2x. Eleven fixes were +> needed, nine of them inside ggml's OpenVINO backend, which is written against +> llama.cpp's graphs and had never seen a vision tower or an action expert - see +> [What had to change](#what-had-to-change). Measured on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10331` @@ -173,6 +176,11 @@ reason in [Known issues](#known-issues). | Evo-1 | 448 | 3,114 ms | 4,523 ms | **574 ms** (5.4x) | not supported | | VLA-Adapter | 224 | 1,228 ms | 1,603 ms | **162 ms** (7.6x) | not supported | | VLA-JEPA | 256 | 1,046 ms | 1,265 ms | **128 ms** (8.2x) | not supported | +| GR00T N1.5 | 224 | not timed | not timed | not timed | not supported | + +GR00T N1.5 is deliberately not timed: it drifts too far from the CPU backend to +report a latency as if the two were doing the same work, and GR00T N1.7 is wrong +outright. The iGPU is the reason to use this backend, and it pays off most where the model is most vision-heavy. The OpenVINO CPU plugin is at best parity with ggml's own @@ -182,13 +190,15 @@ drawing far less power, which is the interesting result for a robot - see the NPU limits under [Known issues](#known-issues). Actions checked against the CPU backend on identical inputs. Both sides are -deterministic run to run, so these are exact, not sampled: +deterministic run to run, so these are exact, not sampled. The SYCL backend's +accepted bar is 2.9e-3 max; the rows are grouped against it. + +**Matches the CPU backend** - safe to treat as a drop-in: | Model | device | max abs deviation | RMS | peak action | |---|---|---:|---:|---:| | SmolVLA | CPU | 1.2e-3 | 1.7e-4 | 0.995 | | SmolVLA | GPU | 1.2e-3 | 1.9e-4 | 0.995 | -| SmolVLA | NPU | 1.6e-2 | 2.1e-3 | 0.995 | | π0.5 | CPU | 8.9e-4 | 8.7e-5 | 0.904 | | π0.5 | GPU | 6.9e-4 | 1.1e-4 | 0.904 | | π0.5 | NPU | 1.6e-3 | 1.9e-4 | 0.904 | @@ -196,20 +206,31 @@ deterministic run to run, so these are exact, not sampled: | Evo-1 | GPU | 2.9e-3 | 4.2e-4 | 0.899 | | VLA-Adapter | CPU | 2.1e-3 | 7.1e-4 | 0.662 | | VLA-Adapter | GPU | 2.9e-3 | 1.1e-3 | 0.662 | -| VLA-JEPA | CPU | 1.2e-2 | 4.1e-3 | 1.145 | -| VLA-JEPA | GPU | 9.1e-3 | 4.5e-3 | 1.145 | -Most of these sit in the same band as the SYCL backend's numbers - kernel -rounding, plus the F16 K/V conversion the SDPA fix introduces. +**Runs, but drifts** - 4-20x the bar above, cause not established. Do not put +these on a robot without checking them against your own policy first: -Two rows are looser and worth naming rather than burying. SmolVLA on the NPU is -an order of magnitude off because the NPU compile config turns on dynamic -quantization; π0.5 on the same device is not, so treat it as a property of that -model on that device. **VLA-JEPA is the loosest CPU/GPU result at ~1% relative, -and it is not diagnosed** - the error is spread evenly across the action vector -rather than sitting in one element, and VLA-JEPA does not use flash attention, so -the SDPA conversion is not the cause. Verify it against your own policy before -trusting VLA-JEPA on this backend. +| Model | device | max abs deviation | RMS | peak action | share of values off by >1e-2 | +|---|---|---:|---:|---:|---:| +| VLA-JEPA | CPU | 1.2e-2 | 4.1e-3 | 1.145 | - | +| VLA-JEPA | GPU | 9.1e-3 | 4.5e-3 | 1.145 | - | +| GR00T N1.5 | CPU | 2.2e-2 | 5.2e-3 | 0.869 | 6.1% | +| GR00T N1.5 | GPU | 5.5e-2 | 1.3e-2 | 0.869 | - | +| SmolVLA | NPU | 1.6e-2 | 2.1e-3 | 0.995 | - | + +**Wrong**: GR00T N1.7, 1.5e0 max on both CPU and GPU, 86% of values off by more +than 1e-2. See [What is left](#what-is-left). + +Most of the first group sits in the same band as the SYCL backend's numbers - +kernel rounding, plus the F16 K/V conversion the SDPA fix introduces. The second +group is the honest open question of this port. SmolVLA on the NPU is explained +(the NPU compile config turns on dynamic quantization, and π0.5 on the same +device stays tight). VLA-JEPA and GR00T N1.5 on CPU and GPU are not explained: +the error is spread across the action vector rather than concentrated, neither +uses flash attention, and GR00T N1.5 does not use mrope either, so the three +usual suspects are all ruled out. GR00T N1.5 is also the one arch that is +markedly worse on the GPU than on the CPU plugin, which points at kernel +precision rather than a structural translation error. ## What had to change @@ -317,8 +338,11 @@ and π0.5 run. The others do not: | Evo-1 | compiler rejects: `Input channels '1025' is not aligned by '16'` | | VLA-Adapter | compiler rejects: `Input channels '261' is not aligned by '16'` | | VLA-JEPA | compiles and runs, returns all `NaN` | +| GR00T N1.5 | NPUW partitioning throws (`partitioning.cpp:1350`) | +| GR00T N1.7 | not attempted - wrong on CPU and GPU already | -The two rejections are Intel's NPU compiler, not vla.cpp: 1025 is Evo-1's 1024 +Four distinct failures, none of them vla.cpp's. The two alignment rejections are +Intel's NPU compiler: 1025 is Evo-1's 1024 patches plus a CLS token, 261 is VLA-Adapter's 256 plus 5, and neither is a multiple of 16. SmolVLA and π0.5 happen to have 16-aligned sequence lengths. The VLA-JEPA NaN is a third failure mode and is not diagnosed. Note that a @@ -358,11 +382,21 @@ vision tower and the same interleaved mrope and is only ~1% off, which points at GR00T N1.7's own action expert rather than anything shared. Not yet diagnosed; the arch stays `-` in the README matrix. +**The drift on VLA-JEPA and GR00T N1.5 is the open question.** Both run, both +are deterministic, and both land 4-20x outside the bar the other four meet. The +three obvious explanations are ruled out: the error is spread across the action +vector rather than concentrated in a few elements, neither uses flash attention +so the SDPA F16 conversion is not implicated, and GR00T N1.5 does not use mrope. +GR00T N1.5 is meaningfully worse on the GPU (5.5e-2) than on the CPU plugin +(2.2e-2), which suggests kernel precision rather than a structural translation +error - but that is a hypothesis, not a finding. Until it is understood, both are +`~` in the README matrix rather than `Y`. + **Untested archs.** π0 and OpenVLA-OFT are untested here for want of a local checkpoint, not because anything is known to block them; both use op sets already covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). GR00T -N1.5 and N1.6 are still being fetched. Treat any untested arch as `-` in the -README matrix until it has actually produced actions. +N1.6 is still being fetched. Treat any untested arch as `-` in the README matrix +until it has actually produced actions. **Splitting across devices.** Intel's own [π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) From 789193531fe5fcd8489f348f8ead2eeafbf46cea Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 20:37:38 +0700 Subject: [PATCH 11/24] measure translation fidelity against an f32 reference, which clears evo-1 and vla-adapter --- docs/backend/ov.md | 135 +++++++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 60 deletions(-) diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 50c29d7..015a25e 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -5,16 +5,20 @@ account of how far it currently runs. Like SYCL, OpenVINO is **not** auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. -> **Status: four architectures match the CPU backend, two more run but drift, -> one is wrong.** SmolVLA, π0.5, Evo-1 and VLA-Adapter agree with the CPU backend -> to 3e-3 or better - the same bar the SYCL backend is held to. VLA-JEPA and -> GR00T N1.5 run to completion but deviate 4-20x more than that, which is not yet -> explained and is enough to matter for control. GR00T N1.7 translates and returns -> plausible-looking actions that are simply wrong. On the Arc B390 iGPU the -> speedup over the native CPU backend runs from 3.1x to 8.2x. Eleven fixes were -> needed, nine of them inside ggml's OpenVINO backend, which is written against -> llama.cpp's graphs and had never seen a vision tower or an action expert - see +> **Status: four architectures translate faithfully, two drift, one is wrong.** +> Evo-1 and VLA-Adapter agree with an F32 CPU reference to six decimal places; +> SmolVLA and π0.5 to under 1e-3. VLA-JEPA (5.5e-3) and GR00T N1.5 (2.7e-2) run to +> completion but sit outside the bar the SYCL backend is held to, for reasons only +> partly established. GR00T N1.7 translates and returns plausible-looking actions +> that are simply wrong. On the Arc B390 iGPU the speedup over the native CPU +> backend runs from 3.1x to 8.2x. Eleven fixes were needed, nine of them inside +> ggml's OpenVINO backend, which is written against llama.cpp's graphs and had +> never seen a vision tower or an action expert - see > [What had to change](#what-had-to-change). +> +> Note the baseline: OpenVINO executes the checkpoint's BF16 weights at F32, so +> compare against `--weight-dtype f32` or you will charge the backend for a +> precision upgrade. See [Picking the right baseline](#picking-the-right-baseline). Measured on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10331` @@ -189,48 +193,58 @@ translation. The NPU beats the CPU backend on the two models it accepts while drawing far less power, which is the interesting result for a robot - see the NPU limits under [Known issues](#known-issues). -Actions checked against the CPU backend on identical inputs. Both sides are -deterministic run to run, so these are exact, not sampled. The SYCL backend's -accepted bar is 2.9e-3 max; the rows are grouped against it. - -**Matches the CPU backend** - safe to treat as a drop-in: - -| Model | device | max abs deviation | RMS | peak action | -|---|---|---:|---:|---:| -| SmolVLA | CPU | 1.2e-3 | 1.7e-4 | 0.995 | -| SmolVLA | GPU | 1.2e-3 | 1.9e-4 | 0.995 | -| π0.5 | CPU | 8.9e-4 | 8.7e-5 | 0.904 | -| π0.5 | GPU | 6.9e-4 | 1.1e-4 | 0.904 | -| π0.5 | NPU | 1.6e-3 | 1.9e-4 | 0.904 | -| Evo-1 | CPU | 2.7e-3 | 3.8e-4 | 0.899 | -| Evo-1 | GPU | 2.9e-3 | 4.2e-4 | 0.899 | -| VLA-Adapter | CPU | 2.1e-3 | 7.1e-4 | 0.662 | -| VLA-Adapter | GPU | 2.9e-3 | 1.1e-3 | 0.662 | - -**Runs, but drifts** - 4-20x the bar above, cause not established. Do not put -these on a robot without checking them against your own policy first: - -| Model | device | max abs deviation | RMS | peak action | share of values off by >1e-2 | -|---|---|---:|---:|---:|---:| -| VLA-JEPA | CPU | 1.2e-2 | 4.1e-3 | 1.145 | - | -| VLA-JEPA | GPU | 9.1e-3 | 4.5e-3 | 1.145 | - | -| GR00T N1.5 | CPU | 2.2e-2 | 5.2e-3 | 0.869 | 6.1% | -| GR00T N1.5 | GPU | 5.5e-2 | 1.3e-2 | 0.869 | - | -| SmolVLA | NPU | 1.6e-2 | 2.1e-3 | 0.995 | - | - -**Wrong**: GR00T N1.7, 1.5e0 max on both CPU and GPU, 86% of values off by more -than 1e-2. See [What is left](#what-is-left). - -Most of the first group sits in the same band as the SYCL backend's numbers - -kernel rounding, plus the F16 K/V conversion the SDPA fix introduces. The second -group is the honest open question of this port. SmolVLA on the NPU is explained -(the NPU compile config turns on dynamic quantization, and π0.5 on the same -device stays tight). VLA-JEPA and GR00T N1.5 on CPU and GPU are not explained: -the error is spread across the action vector rather than concentrated, neither -uses flash attention, and GR00T N1.5 does not use mrope either, so the three -usual suspects are all ruled out. GR00T N1.5 is also the one arch that is -markedly worse on the GPU than on the CPU plugin, which points at kernel -precision rather than a structural translation error. +### Picking the right baseline + +Actions are checked against the CPU backend on identical inputs; both sides are +deterministic, so the numbers are exact rather than sampled. But **which** CPU run +you compare against matters more than it looks. + +OpenVINO folds the checkpoint's BF16 weights in as constants and its CPU plugin +executes them at F32. ggml's CPU backend, on the same checkpoint, keeps them BF16. +So a naive comparison charges the OpenVINO backend for a precision *upgrade*. +Running the reference with `--weight-dtype f32` removes that term and leaves only +what the translation itself contributes: + +| Model | vs BF16 reference (default) | vs F32 reference | what the gap was | +|---|---:|---:|---| +| Evo-1 | 2.7e-3 | **3.6e-6** | almost entirely BF16 vs F32 | +| VLA-Adapter | 2.1e-3 | **2.4e-6** | almost entirely BF16 vs F32 | +| π0.5 | 8.9e-4 | **2.5e-4** | mostly | +| SmolVLA | 1.2e-3 | **9.8e-4** | partly | +| VLA-JEPA | 1.2e-2 | **5.5e-3** | about half | +| GR00T N1.5 | 2.2e-2 | **2.7e-2** | none - not a precision effect | + +Evo-1 and VLA-Adapter agree with an F32 reference to six decimal places, which is +as close to "the translation is exact" as this harness can show. For context, the +CPU backend's own output moves by 2.0e-3 (SmolVLA), 2.7e-3 (Evo-1) or 1.1e-2 +(VLA-JEPA) when you flip that one flag, so the model's intrinsic sensitivity to +precision is the same size as the numbers being reported. + +### Full results + +Against the F32 reference, which is the fidelity number: + +| Model | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | +|---|---:|---:|---:| +| SmolVLA | 9.8e-4 | 1.1e-3 | 1.6e-2 (vs BF16 ref) | +| π0.5 | 2.5e-4 | 5.0e-4 | 1.6e-3 (vs BF16 ref) | +| Evo-1 | 3.6e-6 | 1.2e-3 | not supported | +| VLA-Adapter | 2.4e-6 | 2.7e-3 | not supported | +| VLA-JEPA | 5.5e-3 | 5.4e-3 | returns NaN | +| GR00T N1.5 | 2.7e-2 | 6.1e-2 | NPUW throws | +| GR00T N1.7 | 1.5e0 | 1.5e0 | not attempted | + +The GPU is consistently looser than the CPU plugin because that plugin runs F16 +internally; it stays within the band the SYCL backend is held to (2.9e-3) for +every arch except the two below. + +**Two archs remain unexplained.** VLA-JEPA sits at 5.5e-3, roughly 2x the bar, +with half its original gap accounted for by precision and half not. GR00T N1.5 is +the real outlier: 2.7e-2 that does **not** move with the weight dtype, does not +improve with `--mm-prec f32`, and gets worse on the GPU (6.1e-2). It does not use +mrope or flash attention, so those paths are ruled out. Both stay `~` in the +README matrix. GR00T N1.7 is wrong outright - see +[What is left](#what-is-left). ## What had to change @@ -382,15 +396,16 @@ vision tower and the same interleaved mrope and is only ~1% off, which points at GR00T N1.7's own action expert rather than anything shared. Not yet diagnosed; the arch stays `-` in the README matrix. -**The drift on VLA-JEPA and GR00T N1.5 is the open question.** Both run, both -are deterministic, and both land 4-20x outside the bar the other four meet. The -three obvious explanations are ruled out: the error is spread across the action -vector rather than concentrated in a few elements, neither uses flash attention -so the SDPA F16 conversion is not implicated, and GR00T N1.5 does not use mrope. -GR00T N1.5 is meaningfully worse on the GPU (5.5e-2) than on the CPU plugin -(2.2e-2), which suggests kernel precision rather than a structural translation -error - but that is a hypothesis, not a finding. Until it is understood, both are -`~` in the README matrix rather than `Y`. +**VLA-JEPA and GR00T N1.5 drift, and only half of it is explained.** Measured +against an F32 reference, VLA-JEPA is 5.5e-3 and GR00T N1.5 is 2.7e-2, against a +bar of 2.9e-3. Half of VLA-JEPA's original gap turned out to be the BF16/F32 +baseline; the rest did not. GR00T N1.5's did not move at all with the weight +dtype, nor with `--mm-prec f32`, and it is worse on the GPU (6.1e-2) than the CPU +plugin. Neither arch uses flash attention, and GR00T N1.5 does not use mrope, so +those paths are ruled out. Both stay `~` rather than `Y` until the residual is +understood - the next thing to try is bisecting the graph, since vla.cpp runs the +vision tower and the action head as separate `ggml_backend_graph_compute` calls +and each could be compared in isolation. **Untested archs.** π0 and OpenVLA-OFT are untested here for want of a local checkpoint, not because anything is known to block them; both use op sets already From c5cc004bbe3d1e2314c5de8af4cfff8c7c9de8f8 Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Mon, 31 Aug 2026 21:17:12 +0700 Subject: [PATCH 12/24] add gr00t n1.6 and note that the tighter reference dtype is arch-dependent --- README.md | 10 +++++----- docs/backend/ov.md | 36 +++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 5e4adff..aa82f8a 100644 --- a/README.md +++ b/README.md @@ -312,10 +312,10 @@ Experimental results on other platforms can be found in Support matrix of models (rows) against platforms (columns). Legend: `Y` = supported (released and benchmarked), `~` = in progress, `-` = planned. -OpenVINO runs SmolVLA, π0.5, Evo-1 and VLA-Adapter on Intel CPUs, GPUs and NPUs, -matching the CPU backend to 3e-3; on an Arc B390 iGPU that is 3.1x to 8.2x the -native CPU backend. VLA-JEPA and GR00T N1.5 run but drift further than that and -are marked in progress. Read the known issues in +OpenVINO runs SmolVLA, π0.5, Evo-1, VLA-Adapter and GR00T N1.6 on Intel CPUs, +GPUs and NPUs, matching the CPU backend to 3e-3; on an Arc B390 iGPU that is 3.1x +to 8.2x the native CPU backend. VLA-JEPA and GR00T N1.5 run but drift further and +are marked in progress; GR00T N1.7 is wrong and stays unsupported. Read the known issues in [docs/backend/ov.md](docs/backend/ov.md) before running it - in particular, do not set `GGML_OPENVINO_CACHE_DIR`. @@ -325,7 +325,7 @@ not set `GGML_OPENVINO_CACHE_DIR`. | [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | ~ | | [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | | [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | ~ | -| [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | - | +| [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | | [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | - | | [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | | [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 015a25e..ba43909 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -5,9 +5,9 @@ account of how far it currently runs. Like SYCL, OpenVINO is **not** auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. -> **Status: four architectures translate faithfully, two drift, one is wrong.** +> **Status: five architectures translate faithfully, two drift, one is wrong.** > Evo-1 and VLA-Adapter agree with an F32 CPU reference to six decimal places; -> SmolVLA and π0.5 to under 1e-3. VLA-JEPA (5.5e-3) and GR00T N1.5 (2.7e-2) run to +> SmolVLA and π0.5 to under 1e-3; GR00T N1.6 lands on the bar at 3.0e-3. VLA-JEPA (5.5e-3) and GR00T N1.5 (2.7e-2) run to > completion but sit outside the bar the SYCL backend is held to, for reasons only > partly established. GR00T N1.7 translates and returns plausible-looking actions > that are simply wrong. On the Arc B390 iGPU the speedup over the native CPU @@ -180,6 +180,7 @@ reason in [Known issues](#known-issues). | Evo-1 | 448 | 3,114 ms | 4,523 ms | **574 ms** (5.4x) | not supported | | VLA-Adapter | 224 | 1,228 ms | 1,603 ms | **162 ms** (7.6x) | not supported | | VLA-JEPA | 256 | 1,046 ms | 1,265 ms | **128 ms** (8.2x) | not supported | +| GR00T N1.6 | 224 | 1,276 ms | not timed | **322 ms** (4.0x) | not supported | | GR00T N1.5 | 224 | not timed | not timed | not timed | not supported | GR00T N1.5 is deliberately not timed: it drifts too far from the CPU backend to @@ -202,8 +203,11 @@ you compare against matters more than it looks. OpenVINO folds the checkpoint's BF16 weights in as constants and its CPU plugin executes them at F32. ggml's CPU backend, on the same checkpoint, keeps them BF16. So a naive comparison charges the OpenVINO backend for a precision *upgrade*. -Running the reference with `--weight-dtype f32` removes that term and leaves only -what the translation itself contributes: +Running the reference with `--weight-dtype f32` removes that term. + +The two references bracket the answer, and which one is tighter is arch-dependent +- it turns on how much of a given checkpoint is BF16 in the first place. Report +both and take the smaller as the fidelity figure: | Model | vs BF16 reference (default) | vs F32 reference | what the gap was | |---|---:|---:|---| @@ -212,10 +216,15 @@ what the translation itself contributes: | π0.5 | 8.9e-4 | **2.5e-4** | mostly | | SmolVLA | 1.2e-3 | **9.8e-4** | partly | | VLA-JEPA | 1.2e-2 | **5.5e-3** | about half | +| GR00T N1.6 | **3.0e-3** | 4.7e-3 | none - BF16 is the tighter reference here | | GR00T N1.5 | 2.2e-2 | **2.7e-2** | none - not a precision effect | Evo-1 and VLA-Adapter agree with an F32 reference to six decimal places, which is -as close to "the translation is exact" as this harness can show. For context, the +as close to "the translation is exact" as this harness can show - for those two, +OpenVINO is doing F32 arithmetic and the BF16 comparison was measuring nothing but +the dtype. GR00T N1.6 is the counterexample that stops this being a universal +rule: it lands closer to the BF16 reference, so its checkpoint evidently is not +uniformly BF16 where it matters. For context, the CPU backend's own output moves by 2.0e-3 (SmolVLA), 2.7e-3 (Evo-1) or 1.1e-2 (VLA-JEPA) when you flip that one flag, so the model's intrinsic sensitivity to precision is the same size as the numbers being reported. @@ -226,14 +235,18 @@ Against the F32 reference, which is the fidelity number: | Model | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | |---|---:|---:|---:| -| SmolVLA | 9.8e-4 | 1.1e-3 | 1.6e-2 (vs BF16 ref) | -| π0.5 | 2.5e-4 | 5.0e-4 | 1.6e-3 (vs BF16 ref) | +| SmolVLA | 9.8e-4 | 1.1e-3 | 1.6e-2 | +| π0.5 | 2.5e-4 | 5.0e-4 | 1.6e-3 | | Evo-1 | 3.6e-6 | 1.2e-3 | not supported | | VLA-Adapter | 2.4e-6 | 2.7e-3 | not supported | +| GR00T N1.6 | 3.0e-3 | 3.3e-3 | plugin throws | | VLA-JEPA | 5.5e-3 | 5.4e-3 | returns NaN | | GR00T N1.5 | 2.7e-2 | 6.1e-2 | NPUW throws | | GR00T N1.7 | 1.5e0 | 1.5e0 | not attempted | +GR00T N1.6 sits right on the bar (3.0e-3 against 2.9e-3) rather than comfortably +inside it. + The GPU is consistently looser than the CPU plugin because that plugin runs F16 internally; it stays within the band the SYCL backend is held to (2.9e-3) for every arch except the two below. @@ -353,9 +366,10 @@ and π0.5 run. The others do not: | VLA-Adapter | compiler rejects: `Input channels '261' is not aligned by '16'` | | VLA-JEPA | compiles and runs, returns all `NaN` | | GR00T N1.5 | NPUW partitioning throws (`partitioning.cpp:1350`) | +| GR00T N1.6 | plugin throws (`core.cpp:117`) | | GR00T N1.7 | not attempted - wrong on CPU and GPU already | -Four distinct failures, none of them vla.cpp's. The two alignment rejections are +Five archs, four distinct failures, none of them vla.cpp's. The two alignment rejections are Intel's NPU compiler: 1025 is Evo-1's 1024 patches plus a CLS token, 261 is VLA-Adapter's 256 plus 5, and neither is a multiple of 16. SmolVLA and π0.5 happen to have 16-aligned sequence lengths. The @@ -409,9 +423,9 @@ and each could be compared in isolation. **Untested archs.** π0 and OpenVLA-OFT are untested here for want of a local checkpoint, not because anything is known to block them; both use op sets already -covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). GR00T -N1.6 is still being fetched. Treat any untested arch as `-` in the README matrix -until it has actually produced actions. +covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). Treat +any untested arch as `-` in the README matrix until it has actually produced +actions. **Splitting across devices.** Intel's own [π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) From 57e5f319c26a09858d8f84d2c8357f783be55a47 Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Tue, 1 Sep 2026 10:26:28 +0700 Subject: [PATCH 13/24] translate ggml's gelu as the tanh approximation and give the imrope hunk its missing include --- scripts/patch_ggml_openvino.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index 3821325..3c4bef8 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -441,9 +441,21 @@ namespace ggml { namespace op { +// vla.cpp: ggml's GGML_UNARY_OP_GELU is the *tanh* approximation (its CPU kernel +// additionally reads an fp16 lookup table); ov::op::v7::Gelu defaults to the +// exact erf formulation. Mapping the tanh op onto erf is a real approximation +// mismatch -- small per node, but a vision tower has dozens of them and the +// error compounds through the whole encoder. +static OutputVector translate_gelu_tanh(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, ov::op::GeluApproximationMode::TANH); + return rename_outputs_with_suffix({res}, context.get_name()); +} + // vla.cpp: no ov op takes one input and squares it, so pair the input with // itself rather than route it through Power and a constant exponent. -OutputVector translate_sqr(const NodeContext & context) { +static OutputVector translate_sqr(const NodeContext & context) { num_inputs_check(context, 1, 1); auto input = process_view_input_new(context, 0); auto res = std::make_shared(input, input); @@ -455,10 +467,9 @@ ), ( """ {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input },""", - """ {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input }, - // vla.cpp: ov's Gelu defaults to the exact erf formulation, which is what - // GELU_ERF asks for. GGML_UNARY_OP_GELU above is ggml's tanh - // approximation and keeps the mapping it already had. + """ {"GGML_UNARY_OP_GELU", op::translate_gelu_tanh }, + // vla.cpp: tanh approximation for GELU, exact erf for GELU_ERF. ov's Gelu + // defaults to erf, so the tanh variant must set its mode explicitly. {"GGML_UNARY_OP_GELU_ERF", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input }, {"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input }, @@ -605,6 +616,10 @@ ), ], "ggml/src/ggml-openvino/openvino/translate_session.cpp": [ + ( + '#include "translate_session.h"\n', + '#include "translate_session.h"\n\n#include "ggml.h" // vla.cpp: GGML_ROPE_TYPE_IMROPE\n', + ), ( " auto sin_cos = make_sin_cos(rope_params, inp_pos, rope_freqs_weight);", """ // vla.cpp: rope_params[2] is the mode. The shared precompute never looked at From 39d1eea54d67dbfc3934bb1adfca37c3a91a907e Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Tue, 1 Sep 2026 11:01:07 +0700 Subject: [PATCH 14/24] record seven supported archs, the gelu fix and the corrected gr00t n1.7 findings --- README.md | 13 +--- docs/backend/ov.md | 168 +++++++++++++++++++++++++++------------------ 2 files changed, 104 insertions(+), 77 deletions(-) diff --git a/README.md b/README.md index aa82f8a..0c54cad 100644 --- a/README.md +++ b/README.md @@ -312,26 +312,19 @@ Experimental results on other platforms can be found in Support matrix of models (rows) against platforms (columns). Legend: `Y` = supported (released and benchmarked), `~` = in progress, `-` = planned. -OpenVINO runs SmolVLA, π0.5, Evo-1, VLA-Adapter and GR00T N1.6 on Intel CPUs, -GPUs and NPUs, matching the CPU backend to 3e-3; on an Arc B390 iGPU that is 3.1x -to 8.2x the native CPU backend. VLA-JEPA and GR00T N1.5 run but drift further and -are marked in progress; GR00T N1.7 is wrong and stays unsupported. Read the known issues in -[docs/backend/ov.md](docs/backend/ov.md) before running it - in particular, do -not set `GGML_OPENVINO_CACHE_DIR`. - -| Model | CPU (x86-64 / ARM) | CUDA | SYCL (Intel) | Metal | OpenVINO | +| Model | CPU (x86-64 / ARM) | CUDA | [SYCL (Intel)](docs/backend/sycl.md) | [Metal](docs/backend/metal.md) | [OpenVINO](docs/backend/ov.md) | |---|:--:|:--:|:--:|:--:|:--:| | [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | | [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | ~ | | [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | -| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | ~ | +| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | Y | | [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | | [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | - | | [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | | [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | | [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | | [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | ~ | -| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | ~ | +| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | --- diff --git a/docs/backend/ov.md b/docs/backend/ov.md index ba43909..23926d6 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -5,16 +5,15 @@ account of how far it currently runs. Like SYCL, OpenVINO is **not** auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. -> **Status: five architectures translate faithfully, two drift, one is wrong.** -> Evo-1 and VLA-Adapter agree with an F32 CPU reference to six decimal places; -> SmolVLA and π0.5 to under 1e-3; GR00T N1.6 lands on the bar at 3.0e-3. VLA-JEPA (5.5e-3) and GR00T N1.5 (2.7e-2) run to -> completion but sit outside the bar the SYCL backend is held to, for reasons only -> partly established. GR00T N1.7 translates and returns plausible-looking actions -> that are simply wrong. On the Arc B390 iGPU the speedup over the native CPU -> backend runs from 3.1x to 8.2x. Eleven fixes were needed, nine of them inside -> ggml's OpenVINO backend, which is written against llama.cpp's graphs and had -> never seen a vision tower or an action expert - see -> [What had to change](#what-had-to-change). +> **Status: seven architectures translate faithfully, one is wrong.** SmolVLA, +> π0.5, Evo-1, VLA-Adapter, GR00T N1.5, GR00T N1.6 and VLA-JEPA all agree with a +> CPU-backend reference to 1.3e-3 or better on the OpenVINO CPU plugin - Evo-1 and +> VLA-Adapter to about 3e-6. On the Arc B390 iGPU the speedup over the native CPU +> backend runs from 3.0x to 9.6x. GR00T N1.7 translates and returns +> plausible-looking actions that are simply wrong; it is the one open failure. +> Twelve fixes were needed, ten of them inside ggml's OpenVINO backend, which is +> written against llama.cpp's graphs and had never seen a vision tower or an +> action expert - see [What had to change](#what-had-to-change). > > Note the baseline: OpenVINO executes the checkpoint's BF16 weights at F32, so > compare against `--weight-dtype f32` or you will charge the backend for a @@ -175,17 +174,17 @@ reason in [Known issues](#known-issues). | Model | input | CPU backend | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | |---|---|---:|---:|---:|---:| -| SmolVLA | 512 | 1,364 ms | 1,340 ms | **446 ms** (3.1x) | 1,162 ms | -| π0.5 | 224 | 2,802 ms | 4,285 ms | **641 ms** (4.4x) | 916 ms | -| Evo-1 | 448 | 3,114 ms | 4,523 ms | **574 ms** (5.4x) | not supported | -| VLA-Adapter | 224 | 1,228 ms | 1,603 ms | **162 ms** (7.6x) | not supported | -| VLA-JEPA | 256 | 1,046 ms | 1,265 ms | **128 ms** (8.2x) | not supported | -| GR00T N1.6 | 224 | 1,276 ms | not timed | **322 ms** (4.0x) | not supported | -| GR00T N1.5 | 224 | not timed | not timed | not timed | not supported | - -GR00T N1.5 is deliberately not timed: it drifts too far from the CPU backend to -report a latency as if the two were doing the same work, and GR00T N1.7 is wrong -outright. +| VLA-JEPA | 256 | 1,046 ms | 1,265 ms | **127 ms** (8.2x) | returns NaN | +| GR00T N1.5 | 224 | 1,420 ms | 2,199 ms | **148 ms** (9.6x) | plugin throws | +| VLA-Adapter | 224 | 1,228 ms | 1,603 ms | **161 ms** (7.6x) | not supported | +| GR00T N1.6 | 224 | 1,276 ms | 2,256 ms | **323 ms** (3.9x) | plugin throws | +| SmolVLA | 512 | 1,364 ms | 1,340 ms | **451 ms** (3.0x) | 1,162 ms | +| Evo-1 | 448 | 3,114 ms | 4,523 ms | **563 ms** (5.5x) | not supported | +| π0.5 | 224 | 2,802 ms | 4,285 ms | **683 ms** (4.1x) | 916 ms | +| GR00T N1.7 | 256 | not timed | not timed | not timed | not attempted | + +GR00T N1.7 is not timed because it computes the wrong answer - a latency for work +that is not the same work would be misleading. The iGPU is the reason to use this backend, and it pays off most where the model is most vision-heavy. The OpenVINO CPU plugin is at best parity with ggml's own @@ -209,22 +208,23 @@ The two references bracket the answer, and which one is tighter is arch-dependen - it turns on how much of a given checkpoint is BF16 in the first place. Report both and take the smaller as the fidelity figure: -| Model | vs BF16 reference (default) | vs F32 reference | what the gap was | +| Model | vs BF16 reference | vs F32 reference | tighter reference | |---|---:|---:|---| -| Evo-1 | 2.7e-3 | **3.6e-6** | almost entirely BF16 vs F32 | -| VLA-Adapter | 2.1e-3 | **2.4e-6** | almost entirely BF16 vs F32 | -| π0.5 | 8.9e-4 | **2.5e-4** | mostly | -| SmolVLA | 1.2e-3 | **9.8e-4** | partly | -| VLA-JEPA | 1.2e-2 | **5.5e-3** | about half | -| GR00T N1.6 | **3.0e-3** | 4.7e-3 | none - BF16 is the tighter reference here | -| GR00T N1.5 | 2.2e-2 | **2.7e-2** | none - not a precision effect | +| VLA-Adapter | 2.1e-3 | **2.4e-6** | F32 | +| Evo-1 | 2.7e-3 | **3.5e-6** | F32 | +| π0.5 | 7.7e-4 | **3.5e-5** | F32 | +| VLA-JEPA | 1.1e-2 | **1.1e-4** | F32 | +| GR00T N1.5 | 5.5e-3 | **6.0e-4** | F32 | +| GR00T N1.6 | 5.0e-3 | **1.0e-3** | F32 | +| SmolVLA | **8.9e-4** | 1.3e-3 | BF16 | +| GR00T N1.7 | 1.5e0 | 1.5e0 | neither - it is wrong | Evo-1 and VLA-Adapter agree with an F32 reference to six decimal places, which is as close to "the translation is exact" as this harness can show - for those two, OpenVINO is doing F32 arithmetic and the BF16 comparison was measuring nothing but -the dtype. GR00T N1.6 is the counterexample that stops this being a universal -rule: it lands closer to the BF16 reference, so its checkpoint evidently is not -uniformly BF16 where it matters. For context, the +the dtype. SmolVLA is the counterexample that stops this being a universal rule: it lands +closer to the BF16 reference, so its checkpoint evidently is not uniformly BF16 +where it matters. For context, the CPU backend's own output moves by 2.0e-3 (SmolVLA), 2.7e-3 (Evo-1) or 1.1e-2 (VLA-JEPA) when you flip that one flag, so the model's intrinsic sensitivity to precision is the same size as the numbers being reported. @@ -235,29 +235,27 @@ Against the F32 reference, which is the fidelity number: | Model | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | |---|---:|---:|---:| -| SmolVLA | 9.8e-4 | 1.1e-3 | 1.6e-2 | -| π0.5 | 2.5e-4 | 5.0e-4 | 1.6e-3 | -| Evo-1 | 3.6e-6 | 1.2e-3 | not supported | -| VLA-Adapter | 2.4e-6 | 2.7e-3 | not supported | -| GR00T N1.6 | 3.0e-3 | 3.3e-3 | plugin throws | -| VLA-JEPA | 5.5e-3 | 5.4e-3 | returns NaN | -| GR00T N1.5 | 2.7e-2 | 6.1e-2 | NPUW throws | +| VLA-Adapter | 2.4e-6 | 3.2e-3 | not supported | +| Evo-1 | 3.5e-6 | 1.2e-3 | not supported | +| π0.5 | 3.5e-5 | 4.7e-4 | 9.9e-4 | +| VLA-JEPA | 1.1e-4 | 8.7e-3 | returns NaN | +| GR00T N1.5 | 6.0e-4 | 4.7e-3 | plugin throws | +| GR00T N1.6 | 1.0e-3 | 2.0e-3 | plugin throws | +| SmolVLA | 8.9e-4 | 1.3e-3 | 1.7e-2 | | GR00T N1.7 | 1.5e0 | 1.5e0 | not attempted | -GR00T N1.6 sits right on the bar (3.0e-3 against 2.9e-3) rather than comfortably -inside it. +Every arch except GR00T N1.7 is inside the 2.9e-3 bar on the CPU plugin, most by +one to three orders of magnitude. On the GPU the picture is looser because that +plugin computes in F16: VLA-JEPA (8.7e-3), GR00T N1.5 (4.7e-3) and VLA-Adapter (3.2e-3) sit outside the bar there +even though all three are far inside it on the CPU plugin. Judge translation +fidelity on the CPU plugin; treat the GPU as a separate precision target. -The GPU is consistently looser than the CPU plugin because that plugin runs F16 -internally; it stays within the band the SYCL backend is held to (2.9e-3) for -every arch except the two below. +Two effects explain the residuals that remain. The GPU plugin's F16 arithmetic is +one. The other is SmolVLA on the NPU (1.7e-2), whose compile config turns on +dynamic quantization - π0.5 on the same device stays at 9.9e-4, so that is a +property of the model on that device rather than of the backend. -**Two archs remain unexplained.** VLA-JEPA sits at 5.5e-3, roughly 2x the bar, -with half its original gap accounted for by precision and half not. GR00T N1.5 is -the real outlier: 2.7e-2 that does **not** move with the weight dtype, does not -improve with `--mm-prec f32`, and gets worse on the GPU (6.1e-2). It does not use -mrope or flash attention, so those paths are ruled out. Both stay `~` in the -README matrix. GR00T N1.7 is wrong outright - see -[What is left](#what-is-left). +GR00T N1.7 is wrong outright - see [What is left](#what-is-left). ## What had to change @@ -287,13 +285,14 @@ larger through a model builder that assumes a decoder-only LLM. The literal path is the one that fits a vision tower and an action expert. An explicit setting still wins. -The other seven are in ggml's OpenVINO backend itself, applied by +The other ten are in ggml's OpenVINO backend itself, applied by `scripts/patch_ggml_openvino.py` at configure time. Its docstring carries the detail; in short each narrows an llama.cpp-shaped assumption that is stricter than the ggml contract, or fills a gap: | Fix | What it addresses | |---|---| +| **GELU translated as tanh, not erf** | **assumes ggml's GELU is the exact erf form** | | Intel OpenCL platform selection | assumes the first OpenCL platform is Intel's | | RESHAPE `op_case` guard | assumes a reshape flattening dims 0-2 is the KV-cache flatten | | SDPA K/V converted with Q | assumes K/V arrive as F16 because the KV cache is | @@ -303,7 +302,17 @@ than the ggml contract, or fills a gap: | Missing op translators | RELU, GELU_ERF, NEG, SQR had no table entry | | Naive-path graph cache | that path re-compiled the whole model on every graph_compute | -Three are worth expanding. +Four are worth expanding. + +**The GELU mode** is the highest-yield single fix in the list. ggml's +`GGML_UNARY_OP_GELU` is the *tanh* approximation - its CPU kernel additionally +reads an fp16 lookup table - while `ov::op::v7::Gelu` defaults to the exact erf +formulation, and both ggml GELU ops were mapped onto that default. The error per +node is small, but a Qwen3-VL vision tower contains dozens of them and it +compounds through the encoder. Setting the mode explicitly moved VLA-JEPA from +5.5e-3 to 1.1e-4 (48x) and GR00T N1.5 from 2.7e-2 to 6.0e-4 (45x), turning both +from "runs but drifts" into supported, and improved GR00T N1.6 and π0.5 too. It +is worth upstreaming alongside the position-input fix. **Position inputs** is what carries an arch through to a full prediction. Every tensor feeding a `GGML_OP_ROPE`'s second input was renamed to a single parameter @@ -357,7 +366,7 @@ variable is set. Unverified guess at the cause: the blob key does not capture something that differs between vla.cpp's several graphs, so one graph gets another's blob. In practice, pay the compile once per process and leave it unset. -**The NPU takes two of the five archs, and fails three different ways.** SmolVLA +**The NPU takes two of the eight archs, and fails four different ways.** SmolVLA and π0.5 run. The others do not: | Model | NPU outcome | @@ -410,22 +419,47 @@ vision tower and the same interleaved mrope and is only ~1% off, which points at GR00T N1.7's own action expert rather than anything shared. Not yet diagnosed; the arch stays `-` in the README matrix. -**VLA-JEPA and GR00T N1.5 drift, and only half of it is explained.** Measured -against an F32 reference, VLA-JEPA is 5.5e-3 and GR00T N1.5 is 2.7e-2, against a -bar of 2.9e-3. Half of VLA-JEPA's original gap turned out to be the BF16/F32 -baseline; the rest did not. GR00T N1.5's did not move at all with the weight -dtype, nor with `--mm-prec f32`, and it is worse on the GPU (6.1e-2) than the CPU -plugin. Neither arch uses flash attention, and GR00T N1.5 does not use mrope, so -those paths are ruled out. Both stay `~` rather than `Y` until the residual is -understood - the next thing to try is bisecting the graph, since vla.cpp runs the -vision tower and the action head as separate `ggml_backend_graph_compute` calls -and each could be compared in isolation. +**GR00T N1.7 is the one open failure.** It translates, returns a full action +chunk, and the values are wrong: max|delta| 1.477, rms 8.3e-2 against a peak of +0.948, with 86% of 5280 values off by more than 1e-2. Deterministic, and +identical on the CPU and GPU plugins. Ruled out so far, each with evidence: + +- *Precision.* Identical against BF16 and F32 references. OpenVINO is 1530x less + weight-dtype-sensitive than ggml CPU here, and the model's own bf16/f32 + sensitivity is rms 7.2e-4 against the error's 8.3e-2. +- *Translation-path selection.* The naive path is taken by default and + `is_model_splitted` returns false for every graph of this arch; forcing all + graphs down the decoder-only-LLM path instead changes the answer by 1e-5 while + both remain 1.477 from the reference. +- *Sequence length and token composition.* Swept 3.7x (SEQ 70 to 262) by two + independent routes; relative error stayed within 0.2545-0.2841 and the fraction + off by >1e-2 within 84.7-86.6%. Nothing accumulates. +- *Matmul precision and the compiled-model cache.* Both bitwise no-ops. +- *The GELU mode*, which fixed VLA-JEPA and GR00T N1.5, moves N1.7 by nothing. +- *A missing or unsupported op.* GR00T N1.6 (works) and N1.7 (broken) have the + same 17-op vocabulary, and LM layer 0 is node-for-node identical except that + N1.7's position input is `[4*SEQ]` for IMROPE where N1.6's is `[SEQ]` for NEOX. + VLA-JEPA also uses IMROPE and is now clean, which weakens that lead. + +Three earlier claims about N1.7 turned out to be **measurement artifacts** and +should not be reused. That its first LM block is 73-95% wrong: OpenVINO's +`lm_h_00..03` dumps are bit-exact copies of the *input* arrays and `lm_h_04..15` +are zeros, because the main graph has exactly one real output, `action_pred`. +That `--flash-attn 1` yields 0.948: it actually fails with "Got less inputs than +expected" and returns no actions, and 0.948 is `max|reference|` - the number you +get comparing against nothing. And that honouring `GGML_TENSOR_FLAG_OUTPUT` +changes the answer: same artifact. + +The next step follows from the first artifact. The stage dump cannot see inside +the graph because ggml-openvino writes back only true graph outputs, so either +add a debug mode that materialises selected intermediates as `ov::Result`s, or +bisect with cut-down graphs. **Untested archs.** π0 and OpenVLA-OFT are untested here for want of a local checkpoint, not because anything is known to block them; both use op sets already -covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). Treat -any untested arch as `-` in the README matrix until it has actually produced -actions. +covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). Both +are `~` in the README matrix; treat any untested arch that way until it has +actually produced actions. **Splitting across devices.** Intel's own [π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) From 9bef2b8d0a3361d30a18c77840d808debd4c4626 Mon Sep 17 00:00:00 2001 From: "An T. Le" Date: Tue, 1 Sep 2026 10:50:03 +0700 Subject: [PATCH 15/24] fix the rename that emptied duplicate node names, and key the naive cache on shapes --- .github/workflows/build.yml | 30 ++++++-- .github/workflows/release.yml | 20 +++--- .github/workflows/vla-ci.yml | 4 +- .gitignore | 1 - CHANGELOG.md | 45 ++++++++++++ CMakeLists.txt | 31 +++++--- CONTRIBUTING.md | 5 +- examples/chat/README.md | 6 +- scripts/install_ov.sh | 94 ++++++++++++++++++------ scripts/patch_ggml_openvino.py | 128 +++++++++++++++++++++++++++------ scripts/print_versions.sh | 5 +- src/backend.h | 64 +++++++++++------ src/models/bitvla.cpp | 3 + src/models/dit_common.h | 93 ------------------------ src/models/openvla_oft.cpp | 2 + src/models/pi0.cpp | 2 +- src/models/pi05.cpp | 2 +- src/models/smolvla.cpp | 2 +- src/models/vla_adapter.cpp | 4 +- tests/CMakeLists.txt | 7 ++ tests/test_dit_common.cpp | 2 +- tests/test_graph_names.cpp | 77 ++++++++++++++++++++ 22 files changed, 430 insertions(+), 197 deletions(-) delete mode 100644 src/models/dit_common.h create mode 100644 tests/test_graph_names.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 782e2ef..624768c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: cpp-unit: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 # Compiled directly: pure, no llama.cpp or protobuf/zmq needed. - name: pure unit tests run: | @@ -29,8 +29,8 @@ jobs: py-tooling: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.11' - name: converter remap @@ -41,17 +41,25 @@ jobs: build-gate: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: deps run: | sudo apt-get update -qq sudo apt-get install -y -qq --no-install-recommends \ build-essential cmake git ca-certificates pkg-config \ libzmq3-dev cppzmq-dev libprotobuf-dev protobuf-compiler - - uses: actions/cache@v4 + # Read the pin rather than repeat it: two copies drift and the stale one + # silently reuses the wrong _deps. + - name: read llama.cpp pin + id: pin + run: | + tag=$(grep -m1 'set(VLA_LLAMA_TAG' CMakeLists.txt | grep -oE 'b[0-9]+') + test -n "$tag" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + - uses: actions/cache@v6 with: path: build/_deps - key: llama-b10331-${{ runner.os }} + key: llama-${{ steps.pin.outputs.tag }}-${{ runner.os }} # Everything, not a target list: ctest registers tests this job must build, # and a named list goes stale the next time one is added. - name: build + ctest (CPU, -Wall -Wextra) @@ -59,3 +67,13 @@ jobs: cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=OFF -DVLA_BUILD_TESTS=ON cmake --build build -j"$(nproc)" ctest --test-dir build --output-on-failure + # This job is the only one that fetches llama.cpp, and neither patch script + # runs on a CPU build, so their anchors would otherwise rot unnoticed until + # someone configures a CUDA or OpenVINO tree. Patch a copy: the real one is + # cached. + - name: patch anchors still apply + run: | + cp -r build/_deps/llama-src /tmp/llama-patchtest + python3 scripts/patch_ggml_cuda_ext_hook.py /tmp/llama-patchtest + python3 scripts/patch_ggml_openvino.py /tmp/llama-patchtest + python3 scripts/patch_ggml_openvino.py /tmp/llama-patchtest # idempotent diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d8bc0e3..786f7f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,7 +41,7 @@ jobs: cuda: false runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: deps run: | @@ -75,7 +75,7 @@ jobs: mkdir -p "$out/scripts" && cp scripts/tokenize_prompt.py "$out/scripts/" tar -czf "$out.tar.gz" "$out" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: ${{ matrix.name }} path: '*.tar.gz' @@ -83,7 +83,7 @@ jobs: macos: runs-on: macos-14 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: deps run: brew install cmake zeromq cppzmq protobuf @@ -105,7 +105,7 @@ jobs: find build -name 'default.metallib' -exec cp {} "$out/" \; tar -czf "$out.tar.gz" "$out" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: macos-arm64-metal path: '*.tar.gz' @@ -113,9 +113,9 @@ jobs: docker: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 + - uses: actions/checkout@v7 + - uses: docker/setup-buildx-action@v4 + - uses: docker/login-action@v4 if: startsWith(github.ref, 'refs/tags/') with: registry: ghcr.io @@ -123,7 +123,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: image name run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV" - - uses: docker/build-push-action@v6 + - uses: docker/build-push-action@v7 with: context: . push: ${{ startsWith(github.ref, 'refs/tags/') }} @@ -140,9 +140,9 @@ jobs: steps: # Tarballs only. The docker job also leaves a .dockerbuild build record # artifact behind, and pulling that one fails the whole download. - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: { path: dist, pattern: '{linux,macos}-*', merge-multiple: true } - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: dist/*.tar.gz fail_on_unmatched_files: true diff --git a/.github/workflows/vla-ci.yml b/.github/workflows/vla-ci.yml index 9147eab..57c88bd 100644 --- a/.github/workflows/vla-ci.yml +++ b/.github/workflows/vla-ci.yml @@ -23,10 +23,10 @@ jobs: runs-on: [self-hosted, vla-ci-orchestrator] timeout-minutes: 240 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Sweep all platforms (parallel) + gate run: bash ci/orchestrate.sh all - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: always() with: name: ci-results diff --git a/.gitignore b/.gitignore index 8894aa1..0a78de9 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,6 @@ outputs/ weights/ CLAUDE.md -third_party/llama.cpp/ eval/sim/libero/LIBERO/ eval/sim/libero/libero_uv/ eval/sim/simpler/SimplerEnv/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bb394c..a95f875 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,51 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://keepachangelog.com). +## [Unreleased] + +### Added + +- **OpenVINO backend.** `-DGGML_OPENVINO=ON` runs the archs on Intel CPUs, iGPUs + and NPUs through ggml's OpenVINO backend. SmolVLA, π0.5, Evo-1 and VLA-Adapter + match an F32 CPU reference to 1e-3; on an Arc B390 iGPU that is 3.1x to 8.2x + the native CPU backend. GR00T N1.5/N1.6 and VLA-JEPA run but drift, GR00T N1.7 + is wrong, π0 and OpenVLA-OFT are untested. See `docs/backend/ov.md`. +- `scripts/install_ov.sh` installs the OpenVINO runtime and the Intel GPU/NPU + driver stack on Ubuntu 22.04 and 24.04, with the runtime archive checksummed + against a digest pinned in the script. +- `scripts/patch_ggml_openvino.py` applies eleven fixes to the fetched + ggml-openvino sources at configure time. Each hunk is checked on its own, so a + `build/_deps` patched by an older checkout fails loudly instead of building + something quietly wrong. +- `tests/test_graph_names.cpp` pins `vla::graph_unique_names`. + +### Fixed + +- `graph_unique_names` renamed through `ggml_format_name`, which passes the + tensor's own name to `vsnprintf` as both destination and `%s` source. glibc + empties it, so every duplicate node became the bare string `#`. +- The OpenVINO naive-path compiled-model cache was keyed on node count plus the + first and last node name. Two graphs of the same size collided and the second + ran the first's compiled model. It now also keys on every node's op and shape, + and the map is bounded. +- `GGML_OPENVINO_NAIVE_GRAPH_SIZE` went through `atoi`, so junk parsed to 0 and + sent every graph down the decoder-only-LLM path with nothing said. Empty + environment values no longer count as a setting either. +- `GGML_OPENVINO_CACHE_DIR` is cleared rather than warned about: a warm cache + returns wrong actions, and stderr is not always read. `VLA_ALLOW_OV_CACHE=1` + keeps it. +- `scripts/print_versions.sh` printed `?` for the llama.cpp pin ever since the + tag moved behind `VLA_LLAMA_TAG`. +- The OpenVINO `find_package` failure message was unreachable, sitting after the + fetch whose own `find_package(REQUIRED)` fired first. + +### Changed + +- `src/models/dit_common.h` is gone. It redefined six `vla::` functions that + `src/layers/` already had, with both copies linked into `vla_core`. Every + includer used only `sinusoidal_time_emb` or `build_causal_mask`, so they now + include `layers/embed.h`. Byte-identical across all 11 archs. + ## [0.3.0] - 2026-08-14 Every architecture is byte-identical to 0.2.0 at matching settings. diff --git a/CMakeLists.txt b/CMakeLists.txt index d97a2cb..dafaf5e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,6 +24,14 @@ if(_vla_accel_n GREATER 1) "Enable one accelerator backend at a time; got ${_vla_accel_str}. " "Configure a separate build directory per backend.") endif() +# backend.h calls ggml_backend__init directly, and GGML_BACKEND_DL builds +# the accelerators as loadable modules that are not linked into ggml. Say so here +# rather than at the undefined symbol. +if(_vla_accel_n GREATER 0 AND GGML_BACKEND_DL) + message(FATAL_ERROR + "GGML_BACKEND_DL=ON is not supported with ${_vla_accel}: the backend is " + "built as a module and src/backend.h links its init directly.") +endif() set(LLAMA_BUILD_COMMON ON CACHE BOOL "" FORCE) set(LLAMA_BUILD_TOOLS ON CACHE BOOL "" FORCE) @@ -39,6 +47,17 @@ elseif(GGML_OPENVINO) find_package(Python3 COMPONENTS Interpreter REQUIRED) set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_openvino.py ) + + # Before the fetch: ggml's own ggml-openvino/CMakeLists.txt does + # find_package(OpenVINO REQUIRED ...) and would get there first, so this + # message only reaches anyone if it runs earlier. Same components it asks for. + find_package(OpenVINO QUIET COMPONENTS Runtime Threading) + if(NOT OpenVINO_FOUND) + message(FATAL_ERROR + "GGML_OPENVINO=ON but the OpenVINO runtime was not found. Install it " + "(scripts/install_ov.sh) and 'source /opt/intel/openvino/setupvars.sh' " + "in the shell that configures. See docs/backend/ov.md.") + endif() endif() # Overridable so a regression can be bisected against another tag in a separate # build dir (-DVLA_LLAMA_TAG=b10326) without editing this file. The patch @@ -179,16 +198,8 @@ if(GGML_METAL AND NOT GGML_CUDA AND NOT GGML_SYCL) endif() if(GGML_OPENVINO AND NOT GGML_CUDA AND NOT GGML_SYCL AND NOT GGML_METAL) - # ggml's own OpenVINO backend target finds the toolkit; this only tells the - # archs which branch of the backend.h ladder to compile. Fail early with a - # pointer to setupvars.sh rather than deep inside the fetched tree. - find_package(OpenVINO QUIET COMPONENTS Runtime) - if(NOT OpenVINO_FOUND) - message(FATAL_ERROR - "GGML_OPENVINO=ON but the OpenVINO runtime was not found. Install it " - "(scripts/install_ov.sh) and 'source /opt/intel/openvino/setupvars.sh' " - "in the shell that configures. See docs/backend/ov.md.") - endif() + # Only tells the archs which branch of the backend.h ladder to compile; the + # toolkit was located before the fetch above. target_compile_definitions(vla_core PUBLIC GGML_USE_OPENVINO) endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ce5d34..1fe8e77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,8 +56,9 @@ ckpt) model, `bitvla` for a vision-baked one. Then write `src/models/.cpp`. Before adding a helper, check `src/models/`: `gguf_reader.h` (tensor and KV reads), `vision_common.h` (preprocessing, pixel shuffle), `dual_tower.h` (DINOv2 + SigLIP), -`qwen3vl_vit.h` (Qwen3-VL tower), `dit_common.h` (DiT time embeddings), -`scratch_ctx.h` (compute context reuse), `backend.h` (accelerator selection). +`qwen3vl_vit.h` (Qwen3-VL tower), `layers/embed.h` (time embeddings, causal +mask), `scratch_ctx.h` (compute context reuse), `backend.h` (accelerator +selection). Your loader must fail rather than return a half-built model: check every tensor lookup, and check `real_*_dim <= max_*_dim` (`config_is_sane` in `src/model.cpp` diff --git a/examples/chat/README.md b/examples/chat/README.md index 29aaf4e..3ee40dc 100644 --- a/examples/chat/README.md +++ b/examples/chat/README.md @@ -46,7 +46,7 @@ Convert: SRC=/tmp/smolvlm2-500m-instruct OUT=~/data/$USER/smolvlm2-500m-instruct-gguf mkdir -p "$OUT" -CONV="PYTHONPATH=third_party/llama.cpp/gguf-py python3 third_party/llama.cpp/convert_hf_to_gguf.py" +CONV="PYTHONPATH=build/_deps/llama-src/gguf-py python3 build/_deps/llama-src/convert_hf_to_gguf.py" # text LM -> smolvlm2-500m-instruct-f16.gguf (819 MB) eval $CONV "$SRC" --outtype f16 \ @@ -70,7 +70,7 @@ cmake --build build-cuda --target llama-mtmd-cli ./build-cuda/bin/llama-mtmd-cli \ -m "$OUT/smolvlm2-500m-instruct-f16.gguf" \ --mmproj "$OUT/mmproj-smolvlm2-500m-instruct-f16.gguf" \ - --image third_party/llama.cpp/tools/mtmd/test-1.jpeg -p "Describe this image." --temp 0 + --image build/_deps/llama-src/tools/mtmd/test-1.jpeg -p "Describe this image." --temp 0 ``` > Shortcut: `HuggingFaceTB/SmolVLM2-500M-Video-Instruct` has **byte-identical @@ -123,7 +123,7 @@ python examples/chat/vlm_chat_client.py --addr tcp://localhost:5567 ```bash python examples/chat/vlm_chat_client.py --once \ - --image third_party/llama.cpp/tools/mtmd/test-1.jpeg \ + --image build/_deps/llama-src/tools/mtmd/test-1.jpeg \ -p "Describe this image in detail." -n 200 --temp 0 ``` diff --git a/scripts/install_ov.sh b/scripts/install_ov.sh index 8fba84d..28d450c 100644 --- a/scripts/install_ov.sh +++ b/scripts/install_ov.sh @@ -9,7 +9,7 @@ OS_ID="" OS_VERSION="" log() { - printf '[install_openvino_runetime] %s\n' "$*" + printf '[install_openvino_runtime] %s\n' "$*" } need_cmd() { @@ -19,6 +19,31 @@ need_cmd() { } } +# Digests of the two archives the defaults below pin. These land in /opt under +# sudo, so a bad download is a root-level problem. +OPENVINO_SHA256_2204="d701a115d3dc18088ff75b5b8e67a51fbf780022a3d40ee8ee7f2adfbd9915e6" +OPENVINO_SHA256_2404="6931e5a3c9b1fc9cb170137196df2c40489625703f2d184f511b7add2c110ef8" + +# verify_sha256 . An overridden version has no +# digest here, so fall back to the one the mirror publishes: that catches a +# truncated or corrupted download, not a compromised mirror. +verify_sha256() { + local path="$1" want="$2" url="$3" + if [[ -z "${want}" ]]; then + want="$(curl -fsSL "${url}.sha256" | awk 'NR==1 {print $1}')" + if [[ ! "${want}" =~ ^[0-9a-f]{64}$ ]]; then + echo "Error: no usable checksum published for ${url}" >&2 + exit 1 + fi + log "Version overridden, using the mirror's own checksum." + fi + if ! printf '%s %s\n' "${want}" "${path}" | sha256sum -c - >/dev/null; then + echo "Error: checksum mismatch for ${path}" >&2 + exit 1 + fi + log "Checksum OK: $(basename "${path}")" +} + detect_os() { if [[ ! -f /etc/os-release ]]; then echo "Error: /etc/os-release not found; cannot detect Ubuntu version." >&2 @@ -35,6 +60,14 @@ detect_os() { echo "Error: this installer supports Ubuntu only. Detected ID='${OS_ID:-unknown}'." >&2 exit 1 fi + + # Every package name below is amd64. Say so now, not after the first 404. + local arch + arch="$(uname -m)" + if [[ "${arch}" != "x86_64" ]]; then + echo "Error: Intel publishes these packages for x86_64 only. Detected '${arch}'." >&2 + exit 1 + fi } prepare_common_tools() { @@ -73,7 +106,9 @@ prepare_common_dependencies() { } add_render_group() { - local target_user="${SUDO_USER:-$USER}" + # USER is unbound, not empty, in a container or a cron shell, and set -u would + # abort here with the drivers already installed. + local target_user="${SUDO_USER:-${USER:-}}" if [[ -z "${target_user}" ]]; then return fi @@ -107,14 +142,15 @@ install_gpu_2204() { cd "${download_dir}" for url in "${packages[@]}"; do - wget -c "${url}" + wget --no-continue "${url}" done - wget -c "${crt_base_url}/${checksum_file}" + wget --no-continue "${crt_base_url}/${checksum_file}" - sha256sum -c "${checksum_file}" + sha256sum --ignore-missing -c "${checksum_file}" shopt -s nullglob local artifacts=( *.deb *.ddeb ) + shopt -u nullglob if [[ ${#artifacts[@]} -eq 0 ]]; then echo "Error: no GPU package files found for Ubuntu 22.04." >&2 exit 1 @@ -133,16 +169,13 @@ install_npu_2204() { local level_zero_url="https://github.com/oneapi-src/level-zero/releases/download/v1.24.2/${level_zero_deb}" log "Installing Intel NPU drivers for Ubuntu 22.04..." - sudo dpkg --purge --force-remove-reinstreq \ - intel-driver-compiler-npu \ - intel-fw-npu \ - intel-level-zero-npu \ - intel-level-zero-npu-dbgsym || true - mkdir -p "${download_dir}" cd "${download_dir}" - wget -c "${npu_url}" + # Download before purging. The other order leaves a machine with no NPU driver + # at all if the fetch fails. + wget --no-continue "${npu_url}" + wget --no-continue "${level_zero_url}" tar -xf "${npu_tarball}" mapfile -t npu_debs < <(find . -type f -name '*.deb' ! -name 'level-zero*.deb' | sort) @@ -151,9 +184,13 @@ install_npu_2204() { exit 1 fi - sudo dpkg -i "${npu_debs[@]}" || sudo apt-get install -f -y + sudo dpkg --purge --force-remove-reinstreq \ + intel-driver-compiler-npu \ + intel-fw-npu \ + intel-level-zero-npu \ + intel-level-zero-npu-dbgsym || true - wget -c "${level_zero_url}" + sudo dpkg -i "${npu_debs[@]}" || sudo apt-get install -f -y sudo dpkg -i "${level_zero_deb}" || sudo apt-get install -f -y add_render_group @@ -173,12 +210,17 @@ install_runtime_2204() { local install_dir="${install_root}/openvino_${openvino_version}" local symlink_path="${install_root}/openvino" local archive_path="${download_dir}/openvino_${openvino_version}.tgz" + local expected_sha="" + if [[ "${openvino_version}" == "2025.3" && "${openvino_build}" == "19807.44526285f24" ]]; then + expected_sha="${OPENVINO_SHA256_2204}" + fi log "Installing OpenVINO runtime for Ubuntu 22.04..." sudo mkdir -p "${install_root}" mkdir -p "${download_dir}" curl -fL "${openvino_url}" --output "${archive_path}" + verify_sha256 "${archive_path}" "${expected_sha}" "${openvino_url}" rm -rf "${download_dir:?}/${openvino_dirname}" tar -xf "${archive_path}" -C "${download_dir}" @@ -216,14 +258,15 @@ install_gpu_2404() { cd "${download_dir}" for url in "${packages[@]}"; do - wget -c "${url}" + wget --no-continue "${url}" done - wget -c "${crt_base_url}/${checksum_file}" + wget --no-continue "${crt_base_url}/${checksum_file}" - sha256sum -c "${checksum_file}" + sha256sum --ignore-missing -c "${checksum_file}" shopt -s nullglob local artifacts=( *.deb *.ddeb ) + shopt -u nullglob if [[ ${#artifacts[@]} -eq 0 ]]; then echo "Error: no GPU package files found for Ubuntu 24.04." >&2 exit 1 @@ -247,23 +290,29 @@ install_npu_2404() { ) log "Installing Intel NPU drivers for Ubuntu 24.04..." - sudo dpkg --purge --force-remove-reinstreq "${npu_packages[@]}" || true - mkdir -p "${download_dir}" cd "${download_dir}" - wget -c "${npu_url}" + # Download before purging, so a failed fetch does not leave the machine with + # no NPU driver at all. + wget --no-continue "${npu_url}" tar -xf "${npu_archive}" shopt -s nullglob local debs=( *.deb ) + shopt -u nullglob if [[ ${#debs[@]} -eq 0 ]]; then echo "Error: no Intel NPU .deb packages found for Ubuntu 24.04." >&2 exit 1 fi + sudo dpkg --purge --force-remove-reinstreq "${npu_packages[@]}" || true sudo dpkg -i "${debs[@]}" || sudo apt-get install -f -y + # The Level Zero loader the NPU plugin dlopens. 22.04 needs it from GitHub; + # 24.04 has it in the archive. See docs/backend/ov.md for ZE_ENABLE_ALT_DRIVERS. + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libze1 + add_render_group cd "${SCRIPT_DIR}" @@ -281,12 +330,17 @@ install_runtime_2404() { local install_dir="${install_root}/openvino_${openvino_version}" local symlink_path="${install_root}/openvino" local archive_path="${download_dir}/openvino_${openvino_version}.tgz" + local expected_sha="" + if [[ "${openvino_version}" == "2026.2.1" && "${openvino_build}" == "21919.ede283a88e3" ]]; then + expected_sha="${OPENVINO_SHA256_2404}" + fi log "Installing OpenVINO runtime for Ubuntu 24.04..." sudo mkdir -p "${install_root}" mkdir -p "${download_dir}" curl -fL "${openvino_url}" --output "${archive_path}" + verify_sha256 "${archive_path}" "${expected_sha}" "${openvino_url}" rm -rf "${download_dir:?}/${openvino_dirname}" tar -xf "${archive_path}" -C "${download_dir}" diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index 3c4bef8..eca8ed6 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -18,11 +18,10 @@ ggml-openvino is written against llama.cpp's graphs: one decoder-only transformer, one position input, an F16 KV cache. vla.cpp drives it with vision towers and action experts instead, which is legal ggml but nothing the backend -has seen. Five of the hunks below are places where an llama.cpp-shaped -assumption is narrower than the ggml contract; the sixth is a missing cache on -the path those graphs take. Together they are what lets SmolVLA and pi0.5 run -end to end on the CPU, GPU and NPU plugins. Number 4 is the one that matters -most and the one worth upstreaming. +has seen. Eight of the hunks below narrow an llama.cpp-shaped assumption back to +the ggml contract, one fills a gap in the op table, and two are about the path +those graphs take. Together they are what lets SmolVLA and pi0.5 run end to end +on the CPU, GPU and NPU plugins. Numbers 4 and 5 are the ones worth upstreaming. See docs/backend/ov.md for the measured results and for what is still blocked. @@ -70,9 +69,15 @@ rebuilt the decoder, re-converted the model and called compile_model() on every single ggml_backend_graph_compute. That is the dominant cost once a real graph goes through it: SmolVLA on the CPU plugin drops from 22.7 s to - 1.8 s per prediction with the cache in place. A hit rebinds the cached + 1.4 s per prediction with the cache in place. A hit rebinds the cached decoder to the new graph through the existing update_io(), which is how the dynamic path already handles freshly built tensors. + The key is `naive_key`, not `graph_key`: the latter is n_nodes plus the + first and last node name, which two graphs of the same size can share, and + a compiled model is bound to the shapes it was built for. Reusing one + across a shape change returns another graph's answer with no error, so the + key mixes in every node's op and shape. The map is bounded; see the comment + on the flush. 6. openvino/op_table.cpp - add the four missing op translators. RELU, GELU_ERF, NEG (a unary op) and SQR have no entry in the table, and with no per-op @@ -96,8 +101,8 @@ elementwise ops needs both inputs at the graph's rank or the axis falls outside them. Evo-1 concatenates a CLS weight onto 4-D patch embeddings; SmolVLA concatenates a precomputed time tile onto a 4-D activation. Padding - here is what let vla.cpp drop an arch-specific workaround that had moved - SmolVLA's time tiles into their own buffer. + here is what saves each arch from working around it, e.g. by moving + SmolVLA's time tiles into a buffer of their own. 9. openvino/utils.cpp - bound the interleaved-mrope sector cycle by sections. ggml's IMROPE cycles t/h/w by sector % 3, but only while the sector is @@ -123,10 +128,14 @@ static shapes and no KV-cache inference. That literal path is the one that suits a vision tower, but a vision tower is ~450 nodes. The constant becomes `GGML_OPENVINO_NAIVE_GRAPH_SIZE`; src/backend.h defaults it high - for vla.cpp and an explicit setting still wins. + for vla.cpp and an explicit setting still wins. Parsed with strtol, because + atoi turns junk into 0 and that would send every graph down the LLM builder + with nothing said. -Idempotent - re-running on a patched tree is a no-op, so a reconfigure that -re-populates the FetchContent source dir is safe either way. +Idempotent per hunk, not per file: a tree patched by an older checkout is +missing the hunks added since, and a file-wide marker would skip them and leave +a build that runs and is quietly wrong. Nothing is written until every anchor +has matched, so a mismatch leaves the tree untouched. Usage: scripts/patch_ggml_openvino.py [] """ @@ -134,8 +143,6 @@ import pathlib import sys -MARKER = "vla.cpp:" - HELPER = """// vla.cpp: select the Intel OpenCL platform. With several OpenCL runtimes // installed the first platform is not always Intel's, and the OpenVINO GPU // plugin only accepts an Intel context. Cached: the ICD list cannot change @@ -231,7 +238,7 @@ // paths already do. Conversion plus compile_model dominates a naive call, so // without this every graph_compute pays it again. static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); - const graph_key key(cgraph); + const naive_key key(cgraph); std::shared_ptr entry; bool cache_hit = false; @@ -242,6 +249,12 @@ entry = it->second; cache_hit = true; } else { + // Each entry holds a compiled model, so this cannot grow forever. + // Flush rather than evict: a caller sees a handful of shapes, and an + // LRU is only worth its bookkeeping once that stops being true. + if (r_ctx->naive_cache.size() >= 32) { + r_ctx->naive_cache.clear(); + } entry = std::make_shared(); r_ctx->naive_cache[key] = entry; } @@ -525,6 +538,9 @@ ( "int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {", """bool GgmlOvDecoder::has_multiple_inp_pos() const { + if (m_cgraph == nullptr) { + return false; + } if (m_multi_inp_pos < 0) { std::set seen; for (int i = 0; i < m_cgraph->n_nodes && seen.size() < 2; i++) { @@ -556,12 +572,53 @@ std::shared_ptr infer_request; }; +// vla.cpp: graph_key is {n_nodes, first name, last name}, which two graphs of the +// same size can share. A compiled model is bound to the shapes it was built for, +// so reusing one across a shape change returns another graph's answer with no +// error. Mix the ops and shapes in as well. +inline uint64_t naive_graph_sig(const ggml_cgraph * cgraph) { + uint64_t h = 1469598103934665603ull; + auto mix = [&h](uint64_t v) { h = (h ^ v) * 1099511628211ull; }; + for (int i = 0; i < cgraph->n_nodes; i++) { + const ggml_tensor * node = cgraph->nodes[i]; + mix((uint64_t) node->op); + mix((uint64_t) node->type); + for (int d = 0; d < GGML_MAX_DIMS; d++) { + mix((uint64_t) node->ne[d]); + } + for (int s = 0; s < GGML_MAX_SRC; s++) { + const ggml_tensor * src = node->src[s]; + mix(src ? (uint64_t) src->type + 1 : 0ull); + for (int d = 0; src && d < GGML_MAX_DIMS; d++) { + mix((uint64_t) src->ne[d]); + } + } + } + return h; +} + +struct naive_key { + graph_key base; + uint64_t sig; + + naive_key(const ggml_cgraph * cgraph) : base(cgraph), sig(naive_graph_sig(cgraph)) {} + + bool operator==(const naive_key & other) const { return sig == other.sig && base == other.base; } +}; + +struct naive_key_hash { + size_t operator()(const naive_key & key) const { + size_t h = graph_key_hash{}(key.base); + return h ^ (std::hash{}(key.sig) + 0x9e3779b9 + (h << 6) + (h >> 2)); + } +}; + struct decoder_runtime_ctx {""", ), ( " std::unordered_map, graph_key_hash> decoder_cache;", " std::unordered_map, graph_key_hash> decoder_cache;\n" - " std::unordered_map, graph_key_hash> naive_cache;", + " std::unordered_map, naive_key_hash> naive_cache;", ), ( """ decoder_cache.clear(); @@ -609,9 +666,21 @@ """bool is_naive(ggml_cgraph * cgraph) { // vla.cpp: the literal translation path suits any graph that is not a // decoder-only LLM, so let the caller raise the bar it is chosen under. + // Junk parses to 0 under atoi, which would silently send every graph down + // the LLM builder, so reject anything that is not a whole positive number. static const int naive_graph_size_threshold = [] { const char * env = getenv("GGML_OPENVINO_NAIVE_GRAPH_SIZE"); - return (env && *env) ? atoi(env) : 20; + if (env == nullptr || *env == '\\0') { + return 20; + } + char * end = nullptr; + const long val = strtol(env, &end, 10); + const int n = (int) val; + if (*end != '\\0' || val <= 0 || (long) n != val) { + GGML_LOG_WARN("GGML OpenVINO Backend: ignoring GGML_OPENVINO_NAIVE_GRAPH_SIZE='%s'\\n", env); + return 20; + } + return n; }();""", ), ], @@ -635,24 +704,39 @@ def main() -> int: root = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else ".") + pending = [] for rel, edits in EDITS.items(): src = root / rel if not src.is_file(): print(f"patch_ggml_openvino: {src} not found", file=sys.stderr) return 1 - text = src.read_text() - if MARKER in text: - print(f"patch_ggml_openvino: {rel} already patched") - continue + original = src.read_text() + text = original for anchor, replacement in edits: + # Per hunk, not per file. The hunk list grows between commits, so a + # tree patched by an older checkout still needs the newer ones and a + # file-wide marker would skip them, leaving a build that runs and is + # quietly wrong. + if replacement in text: + continue n = text.count(anchor) if n != 1: - print(f"patch_ggml_openvino: anchor matched {n} times in {rel}, expected 1:\n{anchor}", + print(f"patch_ggml_openvino: anchor matched {n} times in {rel}, expected 1:\n{anchor}\n" + f"A tree patched by an older checkout reads like this. Delete the fetched " + f"llama.cpp (rm -rf /_deps) and reconfigure. If the tree is clean, the " + f"anchor no longer matches VLA_LLAMA_TAG and needs re-targeting.", file=sys.stderr) return 1 - text = text.replace(anchor, replacement) + text = text.replace(anchor, replacement, 1) + if text != original: + pending.append((src, text, rel)) + + # Every anchor matched, so nothing was half-written on the way here. + for src, text, rel in pending: src.write_text(text) print(f"patch_ggml_openvino: patched {rel}") + if not pending: + print("patch_ggml_openvino: already up to date") return 0 diff --git a/scripts/print_versions.sh b/scripts/print_versions.sh index 4681af5..4bc992b 100644 --- a/scripts/print_versions.sh +++ b/scripts/print_versions.sh @@ -79,12 +79,13 @@ echo "- repo HEAD: \`${VLA_HEAD}\` (\`${VLA_DESC}\`)" # ---- llama.cpp ---- echo -echo "### llama.cpp (third_party)" +echo "### llama.cpp (fetched)" echo LLAMA_HEAD=$(git_head_or_dash "$LLAMA_DIR") LLAMA_DESC=$(git_describe_or_dash "$LLAMA_DIR") echo "- HEAD: \`${LLAMA_HEAD}\` (\`${LLAMA_DESC}\`)" -LLAMA_TAG=$(grep -m1 'GIT_TAG' "$ROOT/CMakeLists.txt" | grep -oE 'b[0-9]+' || echo '?') +# GIT_TAG is ${VLA_LLAMA_TAG}, so read the cache variable, not the fetch call. +LLAMA_TAG=$(grep -m1 'set(VLA_LLAMA_TAG' "$ROOT/CMakeLists.txt" | grep -oE 'b[0-9]+' || echo '?') echo "- expected pinned tag (from \`CMakeLists.txt\`): \`${LLAMA_TAG}\`" # ---- GGUFs ---- diff --git a/src/backend.h b/src/backend.h index 4dfd5d1..35b285c 100644 --- a/src/backend.h +++ b/src/backend.h @@ -61,14 +61,18 @@ namespace vla { #if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) // setenv is POSIX. _putenv_s has no "do not overwrite" mode, so check first. +// Empty counts as unset; an empty KEY= in a compose file is not a choice. inline void setenv_default(const char * key, const char * val) { #ifdef _WIN32 size_t len = 0; - if (getenv_s(&len, nullptr, 0, key) == 0 && len > 0) + if (getenv_s(&len, nullptr, 0, key) == 0 && len > 1) // len counts the NUL return; _putenv_s(key, val); #else - setenv(key, val, /*overwrite=*/0); + const char * cur = std::getenv(key); + if (cur && *cur) + return; + setenv(key, val, /*overwrite=*/1); #endif } #endif @@ -104,13 +108,23 @@ inline void graph_unique_names([[maybe_unused]] ggml_cgraph * gf) { // Leafs cannot collide: ggml names an unnamed one "leaf_" as it walks the // graph, and a named one came from the checkpoint. Only results carry a name // derived from their source, so only nodes are checked. - std::unordered_set seen; const int n = ggml_graph_n_nodes(gf); + + std::unordered_set seen; + seen.reserve((size_t) n); + + char buf[GGML_MAX_NAME]; for (int i = 0; i < n; ++i) { ggml_tensor * t = ggml_graph_node(gf, i); if (seen.insert(ggml_get_name(t)).second) continue; - ggml_format_name(t, "%s#%d", ggml_get_name(t), i); - seen.insert(ggml_get_name(t)); + // Local buffer: ggml_format_name would pass t->name as both destination + // and "%s" source, and glibc empties it. Index leads so truncation eats + // the tail, not the distinguisher. k steps by n to stay out of range. + for (int k = i; ; k += n) { + std::snprintf(buf, sizeof(buf), "%d#%s", k, ggml_get_name(t)); + if (seen.insert(buf).second) break; + } + ggml_set_name(t, buf); } #endif } @@ -220,36 +234,44 @@ inline Backend backend_init(const char * tag, int n_threads) { // and one position input. No vla.cpp graph is that shape and all of them // are far larger, so raise the bar the literal path is chosen under; // scripts/patch_ggml_openvino.py is what makes the threshold settable. - // Only a default: an explicit setting wins. ggml caches the value on its - // first OpenVINO entry point, so it has to be set before the init below, - // and call_once because a concurrent model_load would race on the - // environment. + // Only a default: an explicit setting wins. The patched reader latches it + // on the first graph_compute, so here is early enough. call_once because + // a concurrent model_load would race on the environment. static std::once_flag naive_once; std::call_once(naive_once, [] { setenv_default("GGML_OPENVINO_NAIVE_GRAPH_SIZE", "1000000"); }); // ggml exposes OpenVINO as a single device, so VLA_DEVICE does not apply: // the target is chosen by name through GGML_OPENVINO_DEVICE (CPU / GPU / - // NPU) and resolved inside ggml, which prints the winner as "OpenVINO: - // using device X" and quietly falls back to CPU when the requested one is - // not enumerated. Echo what was asked for, so the two lines together say - // whether you got the device you wanted. - // OpenVINO's on-disk blob cache reloads a compiled graph that computes - // the wrong thing here: a cold run is correct, and the next run reading - // those blobs back is not, with no error anywhere. Silently wrong - // actions are the worst failure mode a policy server has, so say so - // loudly rather than let it look like a free speedup. + // NPU) and resolved inside ggml. + // + // OpenVINO's blob cache reloads a graph that computes the wrong thing: + // cold run correct, next run wrong, nothing logged. A warning is no use + // when stderr goes nowhere, so clear it and make opting back in explicit. if (const char * cd = std::getenv("GGML_OPENVINO_CACHE_DIR"); cd && *cd) { + const char * allow = std::getenv("VLA_ALLOW_OV_CACHE"); + const bool keep = allow && allow[0] == '1' && allow[1] == '\0'; std::fprintf(stderr, "%s: WARNING GGML_OPENVINO_CACHE_DIR is set. Reloading cached blobs has been\n" "%s: seen to produce silently incorrect actions on the GPU plugin.\n" - "%s: Unset it unless you have verified the outputs. See docs/backend/ov.md.\n", - tag, tag, tag); + "%s: %s See docs/backend/ov.md.\n", + tag, tag, tag, + keep ? "Kept: VLA_ALLOW_OV_CACHE=1." + : "Ignoring it; set VLA_ALLOW_OV_CACHE=1 to keep it."); + if (!keep) { +#ifdef _WIN32 + _putenv_s("GGML_OPENVINO_CACHE_DIR", ""); +#else + unsetenv("GGML_OPENVINO_CACHE_DIR"); +#endif + } } + // ggml falls back to CPU silently when the requested device is missing, + // so this line is the request. ggml logs what actually ran. const char * want = std::getenv("GGML_OPENVINO_DEVICE"); b.handle = ggml_backend_openvino_init(0); if (b.handle) { - std::printf("%s: backend = OPENVINO (requested device %s)\n", + std::printf("%s: backend = OPENVINO (asked for %s, see ggml's \"using device\" line)\n", tag, (want && *want) ? want : "CPU"); } else { std::fprintf(stderr, "%s: ggml_backend_openvino_init failed; falling back to CPU\n", tag); diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index a749b20..bd67b7a 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -221,6 +221,9 @@ ggml_tensor * build_lm_layer(ggml_context * C, const BitvlaModelArch & m, const ggml_tensor * K = ggml_cont(C, ggml_permute(C, kR, 0, 2, 1, 3)); ggml_tensor * V = ggml_cont(C, ggml_permute(C, v3, 1, 2, 0, 3)); ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + // Unmasked on purpose, same as openvla_oft: BitVLA is fine-tuned with + // OpenVLA-OFT's recipe, which swaps the causal mask for a bidirectional one + // so the action chunk decodes in a single pass. ggml_tensor * att= ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); ggml_tensor * kqv= ggml_mul_mat(C, V, att); ggml_tensor * mer= ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, kqv, 0, 2, 1, 3)), hq, seq); diff --git a/src/models/dit_common.h b/src/models/dit_common.h deleted file mode 100644 index ab6047b..0000000 --- a/src/models/dit_common.h +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2026 VinRobotics -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// DiT head pieces shared verbatim by GR00T N1.5/N1.6/N1.7 and VLA-JEPA. dit_kv -// and build_dit_block take a per-arch struct and stay in their own files. - -#pragma once - -#include "ggml.h" - -#include -#include -#include -#include - -namespace vla { - -// Row id of a stacked [out, in, n_embodiment] weight, applied to x. -inline ggml_tensor * cat_linear(ggml_context * C, ggml_tensor * W3d, ggml_tensor * b2d, int64_t id, ggml_tensor * x) { - const int64_t out = W3d->ne[0], in = W3d->ne[1]; - ggml_tensor * W_id = ggml_view_2d(C, W3d, out, in, W3d->nb[1], (size_t) id * W3d->nb[2]); - ggml_tensor * y = ggml_mul_mat(C, ggml_cont(C, ggml_transpose(C, W_id)), x); - return ggml_add(C, y, ggml_view_1d(C, b2d, out, (size_t) id * b2d->nb[1])); -} - -// Per-block AdaLN. The conditioning vector is (scale, shift) in that order; the -// final projection layer in each arch uses (shift, scale) instead. -inline ggml_tensor * adaln(ggml_context * C, ggml_tensor * x, ggml_tensor * temb, ggml_tensor * lw, ggml_tensor * lb, int64_t dim, float eps) { - ggml_tensor * cond = ggml_add(C, ggml_mul_mat(C, lw, ggml_silu(C, temb)), lb); - ggml_tensor * sc = ggml_view_1d(C, cond, dim, 0), * sh = ggml_view_1d(C, cond, dim, (size_t) dim * sizeof(float)); - ggml_tensor * xn = ggml_norm(C, x, eps); - return ggml_add(C, ggml_add(C, xn, ggml_mul(C, xn, sc)), sh); -} - -// cos first, then sin. Opposite order to action_sinusoid; both match the -// reference and are pinned by tests/test_dit_common.cpp. -inline void timesteps_proj(int64_t bucket, std::vector & out) { - const int64_t half = 128; const float lm = std::log(10000.0f); const float t = (float) bucket; - out.assign(256, 0.0f); - for (int64_t i=0; i & out) { - const int64_t half = dim/2; const float step = std::log(10000.0f)/(float) half; const float t = (float) bucket; - out.assign((size_t) T * dim, 0.0f); - for (int64_t tk=0; tk sinusoidal_time_emb(double t, int64_t dim, double min_p, double max_p) { - const int64_t half = dim/2; - std::vector out(dim); - for (int64_t i=0; i & out) { - out.assign((size_t) seq * seq, 0.0f); - const float NEG = -std::numeric_limits::infinity(); - for (int64_t q=0; q OpenVlaOftModelArch::predict(const Inputs& in) { return {}; } } + // ImageNet constants as bf16 rounds them (0.485 -> 0.484375). The reference + // preprocesses in bf16, so these are the values it actually sees. static const float DMEAN[3]={0.484375f,0.455078125f,0.40625f}, DSTD[3]={0.228515625f,0.2236328125f,0.224609375f}; static const float SMEAN[3]={0.5f,0.5f,0.5f}, SSTD[3]={0.5f,0.5f,0.5f}; diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 824671e..6d9c26d 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -26,7 +26,7 @@ #include "gguf.h" #include "gguf_reader.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "modules/preprocess.h" #include "act_dtype.h" #include "cuda/vla_cuda_ops.h" diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index 06ae20c..cbfc2d7 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -26,7 +26,7 @@ #include "gguf.h" #include "gguf_reader.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "modules/preprocess.h" #include "env_flag.h" diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index 0c67029..4e522c8 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -21,7 +21,7 @@ #include "model.h" #include "modules/preprocess.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "ggml.h" #include "ggml-backend.h" diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index e3b1ad2..12821e0 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -25,7 +25,7 @@ #include "gguf.h" #include "gguf_reader.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "env_flag.h" #include @@ -315,6 +315,8 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { return {}; } } + // ImageNet constants as bf16 rounds them (0.485 -> 0.484375). The reference + // preprocesses in bf16, so these are the values it actually sees. static const float DMEAN[3]={0.484375f,0.455078125f,0.40625f}, DSTD[3]={0.228515625f,0.2236328125f,0.224609375f}; static const float SMEAN[3]={0.5f,0.5f,0.5f}, SSTD[3]={0.5f,0.5f,0.5f}; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5e60cc4..607e79f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,13 @@ target_link_libraries(test_dit_common PRIVATE ggml) target_compile_options(test_dit_common PRIVATE -Wall -Wextra) add_test(NAME dit_common COMMAND test_dit_common) +# Defines GGML_USE_OPENVINO itself so the pass is exercised on any build. +add_executable(test_graph_names test_graph_names.cpp) +target_include_directories(test_graph_names PRIVATE ${CMAKE_SOURCE_DIR}/src) +target_link_libraries(test_graph_names PRIVATE ggml) +target_compile_options(test_graph_names PRIVATE -Wall -Wextra) +add_test(NAME graph_names COMMAND test_graph_names) + add_executable(test_qwen3vl_vit test_qwen3vl_vit.cpp) target_include_directories(test_qwen3vl_vit PRIVATE ${CMAKE_SOURCE_DIR}/src) target_link_libraries(test_qwen3vl_vit PRIVATE ggml) diff --git a/tests/test_dit_common.cpp b/tests/test_dit_common.cpp index 6387f37..1697538 100644 --- a/tests/test_dit_common.cpp +++ b/tests/test_dit_common.cpp @@ -17,7 +17,7 @@ // end-to-end sha oracle cannot do for three of the four archs. The trig orders // are opposite and both match the reference. -#include "models/dit_common.h" +#include "layers/embed.h" #undef NDEBUG // keep assert() live even in Release builds #include diff --git a/tests/test_graph_names.cpp b/tests/test_graph_names.cpp new file mode 100644 index 0000000..0ee8d4f --- /dev/null +++ b/tests/test_graph_names.cpp @@ -0,0 +1,77 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Pins vla::graph_unique_names, which ggml-openvino depends on to tell two +// same-named nodes apart. Defines GGML_USE_OPENVINO so the body compiles on any +// build; nothing here calls into the backend, so only ggml is linked. +// +// The bug this exists to catch: renaming through ggml_format_name passes the +// tensor's own name as the "%s" source, and glibc empties it instead of +// appending, so every duplicate collapsed to "#". + +#define GGML_USE_OPENVINO +#include "backend.h" + +#undef NDEBUG // keep assert() live even in Release builds +#include +#include +#include +#include +#include +#include + +int main() { + ggml_init_params p = { 64u * 1024 * 1024, nullptr, true }; + ggml_context * C = ggml_init(p); + assert(C); + + ggml_tensor * x = ggml_new_tensor_2d(C, GGML_TYPE_F32, 8, 4); + ggml_set_name(x, "x"); + + // Unnamed reshapes all land on the same ggml-derived name. + ggml_cgraph * gf = ggml_new_graph_custom(C, 256, false); + for (int i = 0; i < 6; ++i) + ggml_build_forward_expand(gf, ggml_reshape_2d(C, ggml_scale(C, x, 1.0f + i), 4, 8)); + + const int n = ggml_graph_n_nodes(gf); + assert(n >= 12); + + std::vector before; + for (int i = 0; i < n; ++i) + before.emplace_back(ggml_get_name(ggml_graph_node(gf, i))); + + // Precondition: without the pass the graph really does carry duplicates. + assert(std::set(before.begin(), before.end()).size() < (size_t) n); + + vla::graph_unique_names(gf); + + std::set post; + for (int i = 0; i < n; ++i) { + const std::string nm = ggml_get_name(ggml_graph_node(gf, i)); + assert(!nm.empty()); + assert(post.insert(nm).second); // every node distinct + assert(nm.find(before[i]) != std::string::npos); // and still says what it was + } + + // Idempotent: the graph cache hands the same graph back on every predict. + std::set again; + vla::graph_unique_names(gf); + for (int i = 0; i < n; ++i) + again.insert(ggml_get_name(ggml_graph_node(gf, i))); + assert(again == post); + + ggml_free(C); + std::printf("test_graph_names: OK (%d nodes)\n", n); + return 0; +} From 156bbb66353f1a972861d24bb19864e0671804e8 Mon Sep 17 00:00:00 2001 From: "An T. Le" Date: Tue, 1 Sep 2026 10:56:42 +0700 Subject: [PATCH 16/24] bump llama.cpp to b10729 and re-anchor the openvino patch --- CHANGELOG.md | 8 ++++++ CMakeLists.txt | 2 +- docs/backend/ov.md | 6 ++-- scripts/patch_ggml_openvino.py | 52 +++++++++++----------------------- src/vlm/engine.cpp | 6 ++-- 5 files changed, 33 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a95f875..d2a0f8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke `build/_deps` patched by an older checkout fails loudly instead of building something quietly wrong. - `tests/test_graph_names.cpp` pins `vla::graph_unique_names`. +- CI now checks that both llama.cpp patch scripts still apply, on a copy of + the fetched tree. Neither ran on a CPU build, so their anchors could rot + unnoticed until someone configured a CUDA or OpenVINO tree. ### Fixed @@ -42,6 +45,11 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke ### Changed +- llama.cpp pinned at `b10729`, up from `b10331`. Brings OpenVINO 2026.3.1, the + IM2COL+MatMul to native-convolution fusion, and the `RELU`/`NEG`/`SQR` + translators, which the local patch no longer has to add. Byte-identical on the + CPU backend for all eleven archs. The build.yml cache key now reads the tag out + of `CMakeLists.txt` instead of repeating it. - `src/models/dit_common.h` is gone. It redefined six `vla::` functions that `src/layers/` already had, with both copies linked into `vla_core`. Every includer used only `sinusoidal_time_emb` or `build_causal_mask`, so they now diff --git a/CMakeLists.txt b/CMakeLists.txt index dafaf5e..f3b8db3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,7 @@ endif() # build dir (-DVLA_LLAMA_TAG=b10326) without editing this file. The patch # anchors in scripts/patch_ggml_cuda_ext_hook.py and scripts/patch_ggml_openvino.py # are checked against the default. -set(VLA_LLAMA_TAG "b10331" CACHE STRING "llama.cpp tag to fetch") +set(VLA_LLAMA_TAG "b10729" CACHE STRING "llama.cpp tag to fetch") include(FetchContent) FetchContent_Declare(llama diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 23926d6..32c97d0 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -20,8 +20,10 @@ runtime on the configure line. > precision upgrade. See [Picking the right baseline](#picking-the-right-baseline). Measured on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 -iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10331` -(the tag `CMakeLists.txt` pins), on the checkpoints under `vrfai/` on the Hub. +iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10331`, +on the checkpoints under `vrfai/` on the Hub. `CMakeLists.txt` has since moved to +`b10729`, which is byte-identical on the CPU backend for all eleven archs; the +OpenVINO numbers below have not been re-measured on it. OpenVINO is Intel's inference toolkit; ggml's backend translates a ggml compute graph into an OpenVINO model and hands it to the CPU, GPU or NPU plugin, which diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index eca8ed6..e46201f 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -79,13 +79,15 @@ key mixes in every node's op and shape. The map is bounded; see the comment on the flush. - 6. openvino/op_table.cpp - add the four missing op translators. - RELU, GELU_ERF, NEG (a unary op) and SQR have no entry in the table, and with no per-op - CPU fallback in the core an arch that uses one cannot run at all. Between - them they block every arch except SmolVLA, pi0 and pi0.5. All four map - onto ov ops directly: Relu, Gelu (whose default is the exact erf form - GELU_ERF asks for), Negative, and -- since no single-input ov op squares -- - Multiply with the input on both sides. + 6. openvino/op_table.cpp - get both GELU flavours right. + ggml has two: GGML_UNARY_OP_GELU is the tanh approximation, GGML_UNARY_OP_ + GELU_ERF is the exact one. The table mapped GELU onto ov's Gelu, which + defaults to erf, and had no entry for GELU_ERF at all. So the tanh op was + computed as erf, and an arch using the erf op could not run at all - and + with no per-op CPU fallback in the core, "could not run" means the whole + prediction. Small per node, but a vision tower has dozens and the error + compounds: fixing it is what moved GR00T N1.5 and VLA-JEPA inside the bar. + RELU, NEG and SQR were missing here too; upstream added all three in b10729. 7. openvino/utils.cpp - give a folded weight its full rank before slicing. A ggml tensor that is 2-D folds in as a rank-2 ov constant, which is what a @@ -434,22 +436,12 @@ ], "ggml/src/ggml-openvino/openvino/op_table.cpp": [ ( - """#include -#include -#include - -namespace ov { + """namespace ov { namespace frontend { namespace ggml { std::unordered_map get_supported_ops() {""", - """#include -#include -#include -#include -#include - -namespace ov { + """namespace ov { namespace frontend { namespace ggml { @@ -465,15 +457,6 @@ auto res = std::make_shared(input, ov::op::GeluApproximationMode::TANH); return rename_outputs_with_suffix({res}, context.get_name()); } - -// vla.cpp: no ov op takes one input and squares it, so pair the input with -// itself rather than route it through Power and a constant exponent. -static OutputVector translate_sqr(const NodeContext & context) { - num_inputs_check(context, 1, 1); - auto input = process_view_input_new(context, 0); - auto res = std::make_shared(input, input); - return rename_outputs_with_suffix({res}, context.get_name()); -} } // namespace op std::unordered_map get_supported_ops() {""", @@ -483,10 +466,7 @@ """ {"GGML_UNARY_OP_GELU", op::translate_gelu_tanh }, // vla.cpp: tanh approximation for GELU, exact erf for GELU_ERF. ov's Gelu // defaults to erf, so the tanh variant must set its mode explicitly. - {"GGML_UNARY_OP_GELU_ERF", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input }, - {"GGML_OP_SQR", op::translate_sqr },""", + {"GGML_UNARY_OP_GELU_ERF", op::translate_1to1_match_1_input },""", ), ], "ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp": [ @@ -506,7 +486,7 @@ ], "ggml/src/ggml-openvino/ggml-decoder.h": [ ( - """ std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) { + """ std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) const { if (is_inp_pos(tensor, op)) { return "inp_pos"; }""", @@ -514,7 +494,7 @@ // when the graph really has one. See scripts/patch_ggml_openvino.py. bool has_multiple_inp_pos() const; - std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) { + std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) const { if (is_inp_pos(tensor, op)) { return has_multiple_inp_pos() ? std::string(tensor->name) : std::string("inp_pos"); }""", @@ -641,10 +621,10 @@ ], "ggml/src/ggml-openvino/utils.cpp": [ ( - """ if (!is_model_splitted(cgraph)) { + """ if (!model_is_splitted) { return naive_compute(cgraph, core, device, config); }""", - """ if (!is_model_splitted(cgraph)) { + """ if (!model_is_splitted) { return naive_compute(cgraph, core, device, config, r_ctx); }""", ), diff --git a/src/vlm/engine.cpp b/src/vlm/engine.cpp index e79c147..ab956ff 100644 --- a/src/vlm/engine.cpp +++ b/src/vlm/engine.cpp @@ -135,7 +135,8 @@ bool Engine::decode_image_file(const std::string & path, Image & out) const { if (!loaded()) { return false; } - mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(impl_->vision.get(), path.c_str(), false).bitmap); + mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(impl_->vision.get(), path.c_str(), false, + mtmd_helper_init_opt_default()).bitmap); return bitmap_to_image(bmp, out); } @@ -143,7 +144,8 @@ bool Engine::decode_image_buf(const uint8_t * data, size_t len, Image & out) con if (!loaded() || !data || len == 0) { return false; } - mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(impl_->vision.get(), data, len, false).bitmap); + mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(impl_->vision.get(), data, len, false, + mtmd_helper_init_opt_default()).bitmap); return bitmap_to_image(bmp, out); } From 64f3bbe3afbbbf86204968655fe85b1516d4c808 Mon Sep 17 00:00:00 2001 From: "An T. Le" Date: Tue, 1 Sep 2026 10:59:28 +0700 Subject: [PATCH 17/24] split the ggml-openvino fixes into per-PR branches for upstream --- CHANGELOG.md | 4 + docs/UPSTREAMING.md | 84 +++++++++++++++++ scripts/upstream_split.py | 191 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 279 insertions(+) create mode 100644 docs/UPSTREAMING.md create mode 100755 scripts/upstream_split.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d2a0f8d..1f238d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke - CI now checks that both llama.cpp patch scripts still apply, on a copy of the fetched tree. Neither ran on a CPU build, so their anchors could rot unnoticed until someone configured a CUDA or OpenVINO tree. +- `docs/UPSTREAMING.md` and `scripts/upstream_split.py` regroup the eleven + ggml-openvino fixes into one llama.cpp branch per PR. They are generic + backend defects, not vla.cpp workarounds; landing them upstream removes the + configure-time patch step entirely. ### Fixed diff --git a/docs/UPSTREAMING.md b/docs/UPSTREAMING.md new file mode 100644 index 0000000..741c2e3 --- /dev/null +++ b/docs/UPSTREAMING.md @@ -0,0 +1,84 @@ +# Upstreaming the ggml-openvino fixes + +`scripts/patch_ggml_openvino.py` rewrites nine files inside the fetched llama.cpp +tree at configure time. Every fix in it is a generic ggml-openvino defect, not a +vla.cpp workaround: each narrows an assumption that fits llama.cpp's one +decoder-only graph but is stricter than the ggml contract, or fills a gap in the +op table. None of them needs to live here. + +Landing them upstream deletes the patch step, lets vla.cpp build against stock +llama.cpp, and fixes the same bugs for everyone else translating a graph that is +not a decoder-only LLM - whisper.cpp, embedding models, any vision tower. + +## The series + +A local clone with one branch per PR, each a single commit on `master`: + +``` +~/llama.cpp-upstream +``` + +Regenerate it after editing the patch script: + +```bash +git clone --depth 1 https://github.com/ggml-org/llama.cpp ~/llama.cpp-upstream +python3 scripts/upstream_split.py +``` + +| Branch | What it fixes | Value to upstream | +|---|---|---| +| `openvino-naive-cache` | the naive path re-converts and re-compiles the model on every `graph_compute` | **highest**: 22.7 s to 1.4 s on a SmolVLA graph. Also replaces `graph_key` with a shape-aware `naive_key` for that cache, since a node count plus two names collides | +| `openvino-multiple-inp-pos` | all ROPE position inputs are renamed to one `inp_pos` parameter | **high**: any graph with more than one position tensor fails shape inference today | +| `openvino-view-input-rank` | a folded 2-D weight is sliced at full ggml rank | high: `Axis 2 out of the tensor rank range [-2, 1]` on any fused QKV weight | +| `openvino-concat-rank` | CONCAT cannot broadcast rank | high: same rank-2 constants, different op | +| `openvino-reshape-op-case` | the KV-cache-flatten guard also swallows `ggml_conv_2d`'s kernel reshape | medium | +| `openvino-sdpa-kv-f16` | K/V stay F32 while Q is converted | medium: `Mixed input types are not supported` | +| `openvino-naive-graph-size-env` | the 20-node naive threshold is a `constexpr` | medium: exposes `GGML_OPENVINO_NAIVE_GRAPH_SIZE` | +| `openvino-gelu-erf` | `GGML_UNARY_OP_GELU_ERF` has no table entry | low, trivial | +| `openvino-intel-opencl-platform` | the GPU remote context takes the first OpenCL platform | low, but a hard startup abort when it bites | +| `openvino-imrope-sections` | the IMROPE sector cycle ignores `sections` | low, no measured output change | +| `openvino-imrope-mode` | the shared sin/cos table is built without the imrope flag | low, untested path | + +`RELU`, `NEG` and `SQR` were in the patch too. Upstream added all three in +`b10729`, so they are not in the series. + +## Before submitting + +Read `CONTRIBUTING.md` in llama.cpp. The parts that bite: + +- **One PR per feature.** Hence one branch each; do not squash them. +- **AI usage must be disclosed**, and using AI to write the PR text itself is + prohibited outright. Undisclosed use risks a ban. Write the PR bodies yourself. +- **A modified operator needs `test-backend-ops`.** That covers + `openvino-gelu-erf`, `openvino-sdpa-kv-f16`, `openvino-concat-rank`, + `openvino-view-input-rank` and `openvino-reshape-op-case`. The others touch the + session and cache layers, which `test-backend-ops` does not reach; those need a + before/after run on a real graph in the PR body. +- **A bug fix needs a reproducible case that fails before and passes after.** + Each commit message already names the error string or the timing it fixes. +- New contributors should keep one PR open at a time. `openvino-naive-cache` is + the one to lead with. + +## Not yet written + +**Key the translation map on the tensor, not its name.** `ggml-decoder.cpp` keys +everything on `node->name`. ggml permits duplicate names - it derives a result's +name from its source, so an unnamed `ggml_reshape_2d` is called `" (reshaped)"` - +and llama.cpp only escapes this because it labels every node it builds. Two nodes +sharing a name silently become one and the graph wires the wrong tensor into the +next op. + +`vla::graph_unique_names` in `src/backend.h` works around it from the outside, at +29 call sites. De-duplicating at graph ingest inside `ggml_decoder` would be a few +lines, fix it for every caller, and let vla.cpp delete the helper and all 29 +calls. Worth writing; not in the series because it is new code rather than a fix +already proven on hardware. + +## Status + +The eleven branches carry code that has run on an Intel Core Ultra X7 358H (Arc +B390 iGPU, AI Boost NPU) through vla.cpp's own OpenVINO builds - see +`docs/backend/ov.md` for what that covered. They have **not** been compiled from +these branches: no OpenVINO runtime is installed on the machine that split them, +so `ggml-openvino` does not build there. Build and run each branch before opening +a PR. diff --git a/scripts/upstream_split.py b/scripts/upstream_split.py new file mode 100755 index 0000000..6fd249e --- /dev/null +++ b/scripts/upstream_split.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Split scripts/patch_ggml_openvino.py into one llama.cpp branch per upstream PR. + +Every hunk in the patch script is a generic ggml-openvino fix, so it belongs +upstream rather than in a configure-time rewrite here. This regroups the hunks +into per-PR commits on a clone of llama.cpp master, which is what +docs/UPSTREAMING.md tracks. It only writes to that clone. + + git clone --depth 1 https://github.com/ggml-org/llama.cpp ~/llama.cpp-upstream + python3 scripts/upstream_split.py [] + +Every hunk must land in exactly one PR; the tail of the output says so. +""" +import importlib.util, os, pathlib, subprocess, sys + +REPO = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/llama.cpp-upstream")) +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("p", HERE / "patch_ggml_openvino.py") +m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +E = m.EDITS +D = "ggml/src/ggml-openvino/" + +# branch -> (subject, body, [(file, hunk-index), ...]) +PRS = [ + ("openvino-naive-cache", + "openvino: cache the compiled model on the naive path", + "The dynamic and static paths keep a graph_key-indexed cache of the decoder and\n" + "the compiled infer request. The naive path has none, so every\n" + "ggml_backend_graph_compute rebuilds the decoder, re-converts the model and\n" + "calls compile_model() again. On a graph that is not a decoder-only LLM that is\n" + "the dominant cost: a 512px SmolVLA vision tower plus action expert goes from\n" + "22.7 s to 1.4 s per prediction on the CPU plugin.\n\n" + "A hit rebinds the cached decoder through the existing update_io(), the same way\n" + "the dynamic path handles freshly built tensors.\n\n" + "The cache is keyed on naive_key rather than graph_key. graph_key is n_nodes plus\n" + "the first and last node name, which two graphs of the same size can share, and a\n" + "compiled model is bound to the shapes it was built for, so a collision returns\n" + "another graph's answer with no error. naive_key mixes in every node's op, type\n" + "and shape. The map is bounded and flushed when full.", + [(D+"utils.h",0),(D+"utils.h",1),(D+"utils.h",2),(D+"utils.h",3), + (D+"utils.cpp",0),(D+"utils.cpp",1),(D+"utils.cpp",2)]), + + ("openvino-naive-graph-size-env", + "openvino: make the naive-path graph-size threshold settable", + "Graphs under 20 nodes bypass the LLM decoder and translate literally, with\n" + "static shapes and no KV-cache inference. That literal path is the one that fits\n" + "a graph which is not a decoder-only transformer, but such a graph is routinely\n" + "far larger than 20 nodes: a ViT tower is around 450.\n\n" + "Expose the constant as GGML_OPENVINO_NAIVE_GRAPH_SIZE. Parsed with strtol and\n" + "rejected loudly if it is not a whole positive number, because atoi turns junk\n" + "into 0 and that would send every graph down the LLM builder with nothing said.", + [(D+"utils.cpp",3)]), + + ("openvino-gelu-erf", + "openvino: add the GELU_ERF translator", + "GGML_UNARY_OP_GELU_ERF has no entry in the op table, and with no per-op CPU\n" + "fallback a graph that uses it cannot run. ov's Gelu defaults to the exact erf\n" + "formulation, which is what GELU_ERF asks for; GGML_UNARY_OP_GELU is ggml's tanh\n" + "approximation and keeps the mapping it already has.", + [(D+"openvino/op_table.cpp",0)]), + + ("openvino-multiple-inp-pos", + "openvino: stop distinct position inputs aliasing each other", + "Every tensor feeding a ROPE's second input is renamed to one graph parameter\n" + "called inp_pos, because a llama.cpp graph has exactly one. add_rope_sin_cos()\n" + "then builds a single shared sin/cos table from it.\n\n" + "A graph with several position tensors -- a prefill, a full and a rebased one --\n" + "has them alias each other, and every ROPE takes the table built from whichever\n" + "won. Shape inference then fails:\n\n" + " Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32])\n" + " Argument shapes are inconsistent.\n\n" + "When the graph has more than one, keep each tensor's own name. Nothing is then\n" + "called inp_pos, so add_rope_sin_cos() returns early and the existing fallback in\n" + "translate_rope() builds sin/cos per op from its own position input.\n" + "Single-position graphs are untouched and keep the shared table.", + [(D+"ggml-decoder.h",0),(D+"ggml-decoder.h",1),(D+"ggml-decoder.cpp",1)]), + + ("openvino-reshape-op-case", + "openvino: narrow the RESHAPE op_case 3 guard", + "Case 3 is the KV-cache flatten, [512,1024,1,1] -> [1,524288,1,1], and it emits a\n" + "shape with -1 in dim 2 and 1 in dim 3. Its guard only tests\n" + "src->ne[0]*ne[1]*ne[2] == node->ne[1], which also matches the kernel reshape\n" + "inside ggml_conv_2d ([16,16,3,768] -> [768,768]) and rewrites it to the wrong\n" + "shape. The real case always has node->ne[0] == 1; requiring that sends the conv\n" + "kernel to case 6, the plain reshape.", + [(D+"ggml-decoder.cpp",0)]), + + ("openvino-sdpa-kv-f16", + "openvino: convert K/V to F16 alongside Q in flash_attn_ext", + "The translator converts Q, the mask and the scale to F16 because llama.cpp's KV\n" + "cache already is. A caller that keeps K/V in F32 hits OpenVINO's SDPA rejecting\n" + "mixed input types (\"Mixed input types are not supported\"). Converting K/V too\n" + "matches the precision the translator has already chosen for the other operands.", + [(D+"openvino/op/flash_attn_ext.cpp",0)]), + + ("openvino-view-input-rank", + "openvino: give a folded weight its full rank before slicing", + "A ggml tensor that is 2-D folds in as a rank-2 ov constant, which is what a GEMM\n" + "operand wants, but process_view_input_new() indexes a viewed tensor at its full\n" + "ggml rank, so the slice axis lands outside it:\n\n" + " Slice (Constant blk.0.attn_in.weight[0]:bf16[2688,896], ...)\n" + " Axis 2 out of the tensor rank range [-2, 1].\n\n" + "Reached by viewing Q, K and V out of one fused attn_in weight. Left-pad the\n" + "input with leading 1s, which is the shape ggml gave it anyway.", + [(D+"openvino/utils.cpp",2),(D+"openvino/utils.cpp",3)]), + + ("openvino-concat-rank", + "openvino: align CONCAT input ranks", + "The same rank-2 folded constants reach CONCAT, which unlike the broadcasting\n" + "elementwise ops needs both inputs at the graph's rank or the axis falls outside\n" + "them. Hit by concatenating a CLS weight onto 4-D patch embeddings, and by\n" + "concatenating a precomputed time tile onto a 4-D activation. Left-pad the\n" + "shorter input with leading 1s before picking the axis.", + [(D+"openvino/op/concat.cpp",0),(D+"openvino/op/concat.cpp",1),(D+"openvino/op/concat.cpp",2)]), + + ("openvino-imrope-sections", + "openvino: bound the interleaved-mrope sector cycle by sections", + "ggml's IMROPE cycles t/h/w by sector % 3, but only while the sector is inside\n" + "3 * sections[k]; past that it falls through to the fourth position stream\n" + "(ggml_rope_cache_init in ggml/src/ggml-cpu/ops.cpp). The translator cycled\n" + "unconditionally, so with sections {24,20,20,0} and n_dims 128, sectors 61 and 62\n" + "took h and w instead of e.\n\n" + "No measurable output change on the graphs tested, because the fourth stream\n" + "happened to carry the same positions as the first. Submitted because it removes\n" + "a divergence from the ggml reference, not because a measurement demanded it.", + [(D+"openvino/utils.cpp",0),(D+"openvino/utils.cpp",1)]), + + ("openvino-imrope-mode", + "openvino: pass the rope mode to the shared sin/cos table", + "add_rope_sin_cos() called make_sin_cos() without the imrope flag, so a graph with\n" + "a single position input and interleaved mrope got a table built with the\n" + "plain-rope layout. Silently wrong, not an error. The per-op path in\n" + "translate_rope() already passes the flag; this makes the shared precompute match.", + [(D+"openvino/translate_session.cpp",0)]), + + ("openvino-intel-opencl-platform", + "openvino: select the Intel OpenCL platform for the GPU remote context", + "GGML_OPENVINO_DEVICE=GPU builds an OpenVINO remote context on an OpenCL queue and\n" + "takes the first platform the ICD loader reports. With more than one runtime\n" + "installed (an NVIDIA card beside the Intel iGPU, POCL, Rusticl) that is whichever\n" + "/etc/OpenCL/vendors/*.icd sorted first, and the GPU plugin only accepts an Intel\n" + "context. It aborts at startup with \"Incompatible OpenCL runtime: program is not\n" + "in expected ELF format\".\n\n" + "Select by CL_PLATFORM_VENDOR instead. Single-runtime hosts are unaffected.", + [(D+"ggml-openvino-extra.cpp",0),(D+"ggml-openvino-extra.cpp",1),(D+"ggml-openvino-extra.cpp",2), + (D+"ggml-openvino-extra.cpp",3),(D+"ggml-openvino-extra.cpp",4)]), +] + +def git(*a): + r = subprocess.run(["git", "-C", str(REPO), *a], capture_output=True, text=True) + if r.returncode: sys.exit(f"git {' '.join(a)} failed:\n{r.stderr}") + return r.stdout + +base = git("rev-parse", "HEAD").strip() +used = set() +for branch, subject, body, hunks in PRS: + git("checkout", "-q", "-B", branch, base) + for rel, idx in hunks: + used.add((rel, idx)) + anchor, repl = E[rel][idx] + # The marker is vla.cpp provenance; upstream comments should not carry it. + repl = repl.replace('vla.cpp: ', '') + f = REPO / rel + t = f.read_text() + if repl in t: continue + if t.count(anchor) != 1: + sys.exit(f"{branch}: anchor {idx} in {rel} matched {t.count(anchor)} times") + f.write_text(t.replace(anchor, repl, 1)) + git("add", "-A") + git("commit", "-q", "-m", subject, "-m", body) + print(f"{branch:38s} {git('rev-parse','--short','HEAD').strip()} {len(hunks)} hunk(s)") + +git("checkout", "-q", base) +total = sum(len(v) for v in E.values()) +missing = [(r, i) for r, v in E.items() for i in range(len(v)) if (r, i) not in used] +print(f"\n{len(used)}/{total} hunks assigned to {len(PRS)} branches") +if missing: print("UNASSIGNED:", missing) From 761217857751c0150531ddb71e777f9fdffaff48 Mon Sep 17 00:00:00 2001 From: "An T. Le" Date: Tue, 1 Sep 2026 11:05:27 +0700 Subject: [PATCH 18/24] warn when a build dir still carries the old llama tag --- .gitignore | 3 +-- CMakeLists.txt | 11 ++++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 0a78de9..1d3f9d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Build -build/ -build-*/ +build*/ out/ cmake-build-*/ _workdir* diff --git a/CMakeLists.txt b/CMakeLists.txt index f3b8db3..caf9491 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,16 @@ endif() # build dir (-DVLA_LLAMA_TAG=b10326) without editing this file. The patch # anchors in scripts/patch_ggml_cuda_ext_hook.py and scripts/patch_ggml_openvino.py # are checked against the default. -set(VLA_LLAMA_TAG "b10729" CACHE STRING "llama.cpp tag to fetch") +set(_vla_llama_tag_default "b10729") +set(VLA_LLAMA_TAG "${_vla_llama_tag_default}" CACHE STRING "llama.cpp tag to fetch") +# A cache entry survives an edit to the line above, so an existing build dir keeps +# the tag it was first configured with and quietly builds the wrong llama.cpp. +if(NOT VLA_LLAMA_TAG STREQUAL _vla_llama_tag_default) + message(WARNING + "VLA_LLAMA_TAG is ${VLA_LLAMA_TAG}, not the default ${_vla_llama_tag_default}. " + "If that was not deliberate, this build dir predates the bump: reconfigure with " + "-DVLA_LLAMA_TAG=${_vla_llama_tag_default} or use a fresh one.") +endif() include(FetchContent) FetchContent_Declare(llama From e665a6c5783458154bbb8987712b2c8146b984a9 Mon Sep 17 00:00:00 2001 From: "An T. Le" Date: Tue, 1 Sep 2026 11:25:20 +0700 Subject: [PATCH 19/24] guard bitvla's action slots and fail the load when normalisation stats are unreadable --- CHANGELOG.md | 18 +++++ CMakeLists.txt | 4 + CONTRIBUTING.md | 2 +- docs/ARCHITECTURE.md | 4 +- docs/UPSTREAMING.md | 4 +- scan.sh | 17 ++++ scan2.sh | 3 + scan_results.txt | 162 +++++++++++++++++++++++++++++++++++++ scanner.py | 28 +++++++ scripts/upstream_split.py | 19 +++-- src/loader.cpp | 10 +-- src/model.cpp | 4 +- src/models/bitvla.cpp | 8 ++ src/models/evo1.cpp | 2 +- src/models/pi0.cpp | 31 ++++--- src/models/pi05.cpp | 35 ++++---- src/scratch_ctx.h | 11 ++- tests/CMakeLists.txt | 11 ++- tests/bitvla_gemm_check.cu | 2 +- 19 files changed, 322 insertions(+), 53 deletions(-) create mode 100755 scan.sh create mode 100755 scan2.sh create mode 100644 scan_results.txt create mode 100644 scanner.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f238d1..0e1e1fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,24 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke tag moved behind `VLA_LLAMA_TAG`. - The OpenVINO `find_package` failure message was unreachable, sitting after the fetch whose own `find_package(REQUIRED)` fired first. +- BitVLA indexed its action slots as `seq-2-n_action+i` with no check that the + sequence is long enough. Neither `ggml_get_rows` nor the CUDA gather + bound-checks, so a short prompt read out of bounds and returned it as hidden + states. One guard now covers both LM paths. +- pi0 and pi0.5 fell back to identity normalisation stats on a dimension mismatch + or a short read, and said so on stdout. That returns un-denormalised actions + from a checkpoint that looked fine. Both now fail the load, and the message + goes to stderr - stdout is the action stream `predict_check` diffs. +- `scratch_ctx::reset` ignored an arena larger than the first call's, which would + abort in `ggml_new_tensor` if any call site ever sized one from the input. +- The safetensors arch probe would allocate up to 256 MB for a header it only + substring-searches. Capped at 16 MB. +- The two CUDA targets were the only first-party code built without + `-Wall -Wextra`. +- `tests/bitvla_gemm_check.cu` had no build target and a comment claiming it was + never committed. It builds now, under `GGML_CUDA`. +- Stale references to `vision_common.h` (now `modules/preprocess.h`) and to the + retired `VLA_EVO1_BF16_ACT` switch. ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index caf9491..01e34be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -325,6 +325,10 @@ target_link_libraries(vla-bench PRIVATE vla_core) # Warnings and LTO for our own targets only, never the llama.cpp subtree. set(VLA_FIRST_PARTY_TARGETS vla_core vlm_core vla vla-server vlm-server vla-cli vla-bench) +# The CUDA targets were the only first-party code compiled without warnings. +if(GGML_CUDA) + list(APPEND VLA_FIRST_PARTY_TARGETS bitvla_cuda_kernels vla_cuda_ops) +endif() foreach(tgt IN LISTS VLA_FIRST_PARTY_TARGETS) target_compile_options(${tgt} PRIVATE $<$:-Wall -Wextra>) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1fe8e77..bc5915d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ ckpt) model, `bitvla` for a vision-baked one. 6. `CMakeLists.txt` - add `src/models/.cpp` to `vla_core`. Then write `src/models/.cpp`. Before adding a helper, check -`src/models/`: `gguf_reader.h` (tensor and KV reads), `vision_common.h` +`src/models/`: `gguf_reader.h` (tensor and KV reads), `modules/preprocess.h` (preprocessing, pixel shuffle), `dual_tower.h` (DINOv2 + SigLIP), `qwen3vl_vit.h` (Qwen3-VL tower), `layers/embed.h` (time embeddings, causal mask), `scratch_ctx.h` (compute context reuse), `backend.h` (accelerator diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3ad8cda..0ed9960 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -16,7 +16,7 @@ source is the detail. contexts, vision tower, weights, and compute graph. - `src/models/gguf_reader.h` - the shared GGUF reader (metadata, tensor bytes, on-demand embedding rows). -- `src/models/vision_common.h` - small pure vision helpers (pixel-shuffle, view checks). +- `src/modules/preprocess.h` - small pure vision helpers (pixel-shuffle, view checks). - `src/serving/` - `vla-server` (ZeroMQ + protobuf, action prediction), `vlm-server` (chat), and `vla-cli` (one-shot inference). - `src/kernels/bitvla/` - custom 1.58-bit ternary CUDA kernels for BitVLA. @@ -64,4 +64,4 @@ count; CUDA and Metal run the towers and the transformer on the GPU. Extend the `Arch` enum, declare a `*_create` factory in `arch.h`, implement it under `src/models/`, wire detection and dispatch in `src/model.cpp`, and add a converter in -`scripts/`. Reuse `gguf_reader.h` and `vision_common.h` rather than copying them. +`scripts/`. Reuse `gguf_reader.h` and `modules/preprocess.h` rather than copying them. diff --git a/docs/UPSTREAMING.md b/docs/UPSTREAMING.md index 741c2e3..3f31c7c 100644 --- a/docs/UPSTREAMING.md +++ b/docs/UPSTREAMING.md @@ -34,7 +34,7 @@ python3 scripts/upstream_split.py | `openvino-reshape-op-case` | the KV-cache-flatten guard also swallows `ggml_conv_2d`'s kernel reshape | medium | | `openvino-sdpa-kv-f16` | K/V stay F32 while Q is converted | medium: `Mixed input types are not supported` | | `openvino-naive-graph-size-env` | the 20-node naive threshold is a `constexpr` | medium: exposes `GGML_OPENVINO_NAIVE_GRAPH_SIZE` | -| `openvino-gelu-erf` | `GGML_UNARY_OP_GELU_ERF` has no table entry | low, trivial | +| `openvino-gelu-modes` | ggml's tanh `GELU` is mapped onto ov's erf default, and `GELU_ERF` has no entry at all | **high**: wrong activation on every GELU node. Fixing it moved GR00T N1.5 and VLA-JEPA inside the accuracy bar | | `openvino-intel-opencl-platform` | the GPU remote context takes the first OpenCL platform | low, but a hard startup abort when it bites | | `openvino-imrope-sections` | the IMROPE sector cycle ignores `sections` | low, no measured output change | | `openvino-imrope-mode` | the shared sin/cos table is built without the imrope flag | low, untested path | @@ -50,7 +50,7 @@ Read `CONTRIBUTING.md` in llama.cpp. The parts that bite: - **AI usage must be disclosed**, and using AI to write the PR text itself is prohibited outright. Undisclosed use risks a ban. Write the PR bodies yourself. - **A modified operator needs `test-backend-ops`.** That covers - `openvino-gelu-erf`, `openvino-sdpa-kv-f16`, `openvino-concat-rank`, + `openvino-gelu-modes`, `openvino-sdpa-kv-f16`, `openvino-concat-rank`, `openvino-view-input-rank` and `openvino-reshape-op-case`. The others touch the session and cache layers, which `test-backend-ops` does not reach; those need a before/after run on a real graph in the PR body. diff --git a/scan.sh b/scan.sh new file mode 100755 index 0000000..cdca098 --- /dev/null +++ b/scan.sh @@ -0,0 +1,17 @@ +#!/bin/bash +echo "=== INT OVERFLOWS ===" +rg "(int|int32_t)\s+[a-zA-Z0-9_]+\s*=\s*[a-zA-Z0-9_>.\-]*ne\[[0-3]\]\s*\*" src/ +rg "(int|int32_t)\s+[a-zA-Z0-9_]+\s*=\s*.*\*.*ne\[[0-3]\]" src/ +echo "=== FOPEN WITHOUT FCLOSE ===" +rg -l "fopen" src/ | xargs -I{} bash -c 'grep -q fclose {} || echo {} missing fclose' +echo "=== GGML_NEW_CONTEXT WITHOUT GGML_FREE ===" +rg -l "ggml_init" src/ | xargs -I{} bash -c 'grep -q ggml_free {} || echo {} missing ggml_free' +echo "=== BACKEND BUFFER MEMORY ===" +rg -l "ggml_backend_alloc_buffer" src/ | xargs -I{} bash -c 'grep -q ggml_backend_buffer_free {} || echo {} missing buffer free' +echo "=== SINGLE THREADED LOOPS ===" +rg -i "for.*y.*height.*for.*x.*width" src/ +rg -i "for.*i.*<.*w.*h" src/ +echo "=== F32 ROUND TRIPS ===" +rg "ggml_cast.*F32" src/ +rg "ggml_cpy.*F32" src/ +echo "=== UNCHECKED TENSOR SHAPES ===" diff --git a/scan2.sh b/scan2.sh new file mode 100755 index 0000000..e1b5d30 --- /dev/null +++ b/scan2.sh @@ -0,0 +1,3 @@ +rg -n "\bmalloc\(" src/ +rg -n "\bnew\b" src/ +rg -n "\bfopen\(" src/ diff --git a/scan_results.txt b/scan_results.txt new file mode 100644 index 0000000..213f55d --- /dev/null +++ b/scan_results.txt @@ -0,0 +1,162 @@ + +--- Checking src/model.cpp --- +BUGS_FILE src/model.cpp:46 | gguf_context * gctx = gguf_init_from_file(path.c_str(), p); + +--- Checking src/loader.cpp --- + +--- Checking src/vla_c_api.cpp --- + +--- Checking src/options.cpp --- + +--- Checking src/vlm/engine.cpp --- + +--- Checking src/serving/server.cpp --- + +--- Checking src/serving/vla-cli.cpp --- +BUGS_FILE src/serving/vla-cli.cpp:172 | FILE * fp = popen(cmd.c_str(), "r"); + +--- Checking src/serving/vlm-server.cpp --- + +--- Checking src/serving/vla-bench.cpp --- + +--- Checking src/models/bitvla.cpp --- +BUGS_FILE src/models/bitvla.cpp:596 | if (!g.open(ckpt_path)) +BUGS_FILE src/models/bitvla.cpp:607 | if (!m->emb_reader.open(ckpt_path)) +BUGS_FILE src/models/bitvla.cpp:1074 | FILE* f = std::fopen(path.c_str(), "wb"); +BUGS_FILE src/models/bitvla.cpp:1084 | FILE* f = std::fopen(path.c_str(), "a"); +BUGS_FILE src/models/bitvla.cpp:1094 | FILE* f = std::fopen(p.c_str(), "w"); if (f) std::fclose(f); + +--- Checking src/models/vla_jepa.cpp --- +BUGS_FILE src/models/vla_jepa.cpp:227 | if (!g.open(ckpt_path)) +BUGS_FILE src/models/vla_jepa.cpp:298 | if (!io.open(gguf_path)) { +BUGS_FILE src/models/vla_jepa.cpp:299 | std::fprintf(stderr, "vla(vla_jepa): build_caches: io.open(%s) failed\n", gguf_path.c_str()); +BUGS_FILE src/models/vla_jepa.cpp:337 | FILE * fp = std::fopen(path, "wb"); if (fp) { +BUGS_FILE src/models/vla_jepa.cpp:356 | FILE * fp = std::fopen(cond_file, "rb"); +BUGS_FILE src/models/vla_jepa.cpp:376 | FILE * fp = std::fopen(patches_file, "rb"); +BUGS_FILE src/models/vla_jepa.cpp:438 | if (dump_prefix) { char nm[32]; std::snprintf(nm, sizeof(nm), "vit_view%lld", (long long) v); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump_prefix, nm, (long long) H, (long long) K); FILE * fp = std::fopen(path, "wb"); if (fp) { std::fwrite(img_emb_host.data()+v * K * H, sizeof(float), (size_t) K * H, fp); std::fclose(fp); } } +PERF_LOOP src/models/vla_jepa.cpp:464 | { int64_t k = 0; for (int64_t p=0; pio.open(ckpt_path)) + +--- Checking src/models/evo1.cpp --- +BUGS_FILE src/models/evo1.cpp:371 | if (!m->io.open(ckpt_path)) + +--- Checking src/models/vla_adapter.cpp --- +BUGS_FILE src/models/vla_adapter.cpp:191 | if (!g.open(ckpt_path)) + +--- Checking src/models/pi0.cpp --- +BUGS_FILE src/models/pi0.cpp:359 | if (!m->io.open(ckpt_path)) + +--- Checking src/models/gr00tn1d7.cpp --- +BUGS_FILE src/models/gr00tn1d7.cpp:251 | if (!g.open(ckpt_path)) +BUGS_FILE src/models/gr00tn1d7.cpp:323 | if (!io.open(gguf_path)) { +BUGS_FILE src/models/gr00tn1d7.cpp:324 | std::fprintf(stderr, "vla(gr00tn1d7): build_caches: io.open(%s) failed\n", gguf_path.c_str()); +BUGS_FILE src/models/gr00tn1d7.cpp:719 | FILE * fp = std::fopen(path, "wb"); +BUGS_FILE src/models/gr00tn1d7.cpp:744 | FILE * fp = std::fopen(path, "wb"); + +--- Checking src/models/gr00tn1d5.cpp --- +BUGS_FILE src/models/gr00tn1d5.cpp:216 | if (!m->io.open(ckpt_path)) + +--- Checking src/models/smolvla.cpp --- +BUGS_FILE src/models/smolvla.cpp:64 | bool open(const std::string & path) { +BUGS_FILE src/models/smolvla.cpp:65 | file.open(path, std::ios::binary); +BUGS_FILE src/models/smolvla.cpp:157 | bool open(const std::string & path) { +BUGS_FILE src/models/smolvla.cpp:161 | gctx = gguf_init_from_file(path.c_str(), p); +BUGS_FILE src/models/smolvla.cpp:163 | std::fprintf(stderr, "vla: gguf_init_from_file failed for %s\n", path.c_str()); +BUGS_FILE src/models/smolvla.cpp:166 | fp = std::fopen(path.c_str(), "rb"); +BUGS_FILE src/models/smolvla.cpp:168 | std::fprintf(stderr, "vla: fopen failed for %s\n", path.c_str()); +BUGS_FILE src/models/smolvla.cpp:527 | if (!st.open(sf_path)) { +BUGS_FILE src/models/smolvla.cpp:952 | if (!gst.open(ckpt_path)) { +BUGS_FILE src/models/smolvla.cpp:1010 | if (!st.open(ckpt_path)) { + +--- Checking src/models/openvla_oft.cpp --- +BUGS_FILE src/models/openvla_oft.cpp:165 | if (!g.open(ckpt_path)) + +--- Checking src/models/pi05.cpp --- +BUGS_FILE src/models/pi05.cpp:399 | if (!m->io.open(ckpt_path)) + +--- Checking src/modules/encoder.cpp --- + +--- Checking src/modules/dit_head.cpp --- + +--- Checking src/modules/siglip_vit.cpp --- + +--- Checking src/modules/action_expert.cpp --- + +--- Checking src/modules/qwen3_lm.cpp --- + +--- Checking src/modules/prompt.cpp --- + +--- Checking src/act_dtype.h --- + +--- Checking src/arch.h --- +BUGS_MEM src/arch.h:21 | * architecture means: extend the @ref vla::Arch enum, declare a new factory + +--- Checking src/env_flag.h --- + +--- Checking src/backend.h --- + +--- Checking src/gguf_reader.h --- +BUGS_FILE src/gguf_reader.h:52 | bool open(const std::string & path) { +BUGS_FILE src/gguf_reader.h:56 | gctx = gguf_init_from_file(path.c_str(), p); +BUGS_FILE src/gguf_reader.h:58 | std::fprintf(stderr, "vla(%s): gguf_init_from_file failed for %s\n", arch, path.c_str()); +BUGS_FILE src/gguf_reader.h:61 | fp = std::fopen(path.c_str(), "rb"); +BUGS_FILE src/gguf_reader.h:63 | std::fprintf(stderr, "vla(%s): fopen failed for %s\n", arch, path.c_str()); + +--- Checking src/model.h --- + +--- Checking src/options.h --- + +--- Checking src/loader.h --- + +--- Checking src/scratch_ctx.h --- + +--- Checking src/vlm/engine.h --- + +--- Checking src/serving/hf_fetch.h --- + +--- Checking src/models/dit_common.h --- + +--- Checking src/cuda/vla_cuda_ops.h --- + +--- Checking src/kernels/bitvla/bitvla_fp32head_cuda.h --- + +--- Checking src/kernels/bitvla/bitnet_kernels.h --- +API_HEADER src/kernels/bitvla/bitnet_kernels.h:1 | Missing pragma once + +--- Checking src/kernels/bitvla/bitvla_vit_cuda.h --- + +--- Checking src/kernels/bitvla/bitvla_lm_cuda.h --- + +--- Checking src/modules/encoder.h --- + +--- Checking src/modules/dit_head.h --- + +--- Checking src/modules/qwen3_lm.h --- + +--- Checking src/modules/dual_tower.h --- + +--- Checking src/modules/gemma_expert.h --- + +--- Checking src/modules/action_expert.h --- + +--- Checking src/modules/siglip_vit.h --- + +--- Checking src/modules/prompt.h --- + +--- Checking src/modules/preprocess.h --- + +--- Checking src/modules/qwen3vl_vit.h --- + +--- Checking src/layers/norm.h --- + +--- Checking src/layers/linear.h --- + +--- Checking src/layers/attn.h --- + +--- Checking src/layers/embed.h --- + +--- Checking src/layers/ffn.h --- + +--- Checking src/layers/rope.h --- diff --git a/scanner.py b/scanner.py new file mode 100644 index 0000000..0b7a7d7 --- /dev/null +++ b/scanner.py @@ -0,0 +1,28 @@ +import os, glob + +def check_file(path): + with open(path, 'r') as f: + lines = f.readlines() + + print(f"\n--- Checking {path} ---") + for i, line in enumerate(lines): + line_num = i + 1 + # BUGS + if "gguf_init_from_file" in line or "fopen" in line or "open(" in line: + print(f"BUGS_FILE {path}:{line_num} | {line.strip()}") + if "malloc" in line or "new " in line: + if not "delete" in "".join(lines) and not "free" in "".join(lines): + print(f"BUGS_MEM {path}:{line_num} | {line.strip()}") + + # PERFORMANCE + if "for (" in line or "while (" in line: + if "get_f32" in line or "set_f32" in line or "memcpy" in line: + print(f"PERF_LOOP {path}:{line_num} | {line.strip()}") + + # API/BUILD + if "pragma once" not in "".join(lines) and path.endswith(".h"): + print(f"API_HEADER {path}:{line_num} | Missing pragma once") + break + +for f in glob.glob("src/**/*.cpp", recursive=True) + glob.glob("src/**/*.h", recursive=True): + check_file(f) diff --git a/scripts/upstream_split.py b/scripts/upstream_split.py index 6fd249e..8a820e6 100755 --- a/scripts/upstream_split.py +++ b/scripts/upstream_split.py @@ -65,13 +65,16 @@ "into 0 and that would send every graph down the LLM builder with nothing said.", [(D+"utils.cpp",3)]), - ("openvino-gelu-erf", - "openvino: add the GELU_ERF translator", - "GGML_UNARY_OP_GELU_ERF has no entry in the op table, and with no per-op CPU\n" - "fallback a graph that uses it cannot run. ov's Gelu defaults to the exact erf\n" - "formulation, which is what GELU_ERF asks for; GGML_UNARY_OP_GELU is ggml's tanh\n" - "approximation and keeps the mapping it already has.", - [(D+"openvino/op_table.cpp",0)]), + ("openvino-gelu-modes", + "openvino: map GELU to tanh and add GELU_ERF", + "ggml has two GELUs: GGML_UNARY_OP_GELU is the tanh approximation and\n" + "GGML_UNARY_OP_GELU_ERF is the exact one. The table maps GELU onto ov's Gelu,\n" + "which defaults to erf, and has no entry for GELU_ERF at all. So the tanh op is\n" + "computed as erf, and a graph using the erf op cannot run.\n\n" + "Small per node, but a vision tower has dozens and it compounds: on a ggml graph\n" + "with a ViT encoder, fixing the mode moved two models from visibly wrong output\n" + "to within 1.3e-3 of the CPU-backend reference.", + [(D+"openvino/op_table.cpp",0),(D+"openvino/op_table.cpp",1)]), ("openvino-multiple-inp-pos", "openvino: stop distinct position inputs aliasing each other", @@ -145,7 +148,7 @@ "a single position input and interleaved mrope got a table built with the\n" "plain-rope layout. Silently wrong, not an error. The per-op path in\n" "translate_rope() already passes the flag; this makes the shared precompute match.", - [(D+"openvino/translate_session.cpp",0)]), + [(D+"openvino/translate_session.cpp",0),(D+"openvino/translate_session.cpp",1)]), ("openvino-intel-opencl-platform", "openvino: select the Intel OpenCL platform for the GPU remote context", diff --git a/src/loader.cpp b/src/loader.cpp index 11dc668..8e231c0 100644 --- a/src/loader.cpp +++ b/src/loader.cpp @@ -162,21 +162,21 @@ bool WeightLoader::upload(ggml_backend_t backend, ggml_backend_buffer_t * out_bu } for (const Fused & f : fused_) { - std::vector buf; + std::vector parts; for (const std::string & s : f.srcs) { std::vector b = g_.read_convert(s.c_str(), f.dst->type); if (b.empty()) { std::fprintf(stderr, "vla(%s): fused fill: read %s failed\n", arch_, s.c_str()); return false; } - buf.insert(buf.end(), b.begin(), b.end()); + parts.insert(parts.end(), b.begin(), b.end()); } - if (buf.size() != ggml_nbytes(f.dst)) { + if (parts.size() != ggml_nbytes(f.dst)) { std::fprintf(stderr, "vla(%s): fused fill: %s size %zu vs %zu\n", - arch_, ggml_get_name(f.dst), buf.size(), ggml_nbytes(f.dst)); + arch_, ggml_get_name(f.dst), parts.size(), ggml_nbytes(f.dst)); return false; } - ggml_backend_tensor_set(f.dst, buf.data(), 0, buf.size()); + ggml_backend_tensor_set(f.dst, parts.data(), 0, parts.size()); } return true; } diff --git a/src/model.cpp b/src/model.cpp index b700ec3..f379a9c 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -156,7 +156,9 @@ bool detect_arch_safetensors(const std::string& path, Arch* out) { return false; uint64_t header_size = 0; f.read(reinterpret_cast(&header_size), sizeof(header_size)); - if (!f || header_size == 0 || header_size > (1u << 28)) + // The probe only substring-searches for a namespace prefix, so it never + // needs more than a real header: the largest in tree is a couple of MB. + if (!f || header_size == 0 || header_size > (1u << 24)) return false; std::string header(header_size, '\0'); f.read(header.data(), header_size); diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index bd67b7a..71df94f 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -1271,6 +1271,14 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(bitvla): seq=%lld > lm_max_pos=%lld\n", (long long) seq, (long long) lm_max_pos); return {}; } + // Both LM paths index the action slots as seq-2-n_action+i and neither + // ggml_get_rows nor the CUDA gather bound-checks, so a short sequence would + // read out of bounds and come back as plausible hidden states. + if (seq < n_action+2) { + std::fprintf(stderr, "vla(bitvla): seq=%lld too short for %lld action slots\n", + (long long) seq, (long long) n_action); + return {}; + } std::vector inputs_embeds((size_t) seq * hidden_l, 0.0f); diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index d59128d..1721fb6 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -85,7 +85,7 @@ struct Evo1ModelArch : public ModelArchBase { ggml_backend_buffer_t weight_buf = nullptr; ggml_type matmul_type = GGML_TYPE_BF16; // Activation dtype carried between ops. F32 by default; BF16 under - // VLA_EVO1_BF16_ACT, which removes the per-GEMM F32<->BF16 round trip ggml + // --act-dtype bf16, which removes the per-GEMM F32<->BF16 round trip ggml // pays when BF16 weights meet F32 activations. See mm_act/as_type below. ggml_type act_type = GGML_TYPE_F32; diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 6d9c26d..43c8076 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -303,28 +303,33 @@ bool load_stats(gguf_reader & g, Pi0ModelArch & m) { m.state_std .assign(cfg.real_state_dim, 1.f); m.action_mean.assign(cfg.real_action_dim, 0.f); m.action_std .assign(cfg.real_action_dim, 1.f); + // Absent stats are a valid checkpoint: identity, carry on. Stats that are + // present but unreadable are not - falling back to identity there hands back + // un-denormalised actions with nothing in the log. Note stderr, not stdout: + // stdout is the action stream tests/predict_check.cpp diffs. auto read1d = [&](const char * name, std::vector & dst) { const ggml_tensor * t = g.meta(name); if (!t) { - std::printf("vla(pi0): %s missing - identity\n", name); - return; + std::fprintf(stderr, "vla(pi0): %s missing - identity\n", name); + return true; } if (t->ne[0] != (int64_t) dst.size()) { - std::printf("vla(pi0): %s dim mismatch - identity\n", name); - return; + std::fprintf(stderr, "vla(pi0): %s is %lld wide, expected %zu\n", + name, (long long) t->ne[0], dst.size()); + return false; } - const std::vector identity = dst; if (!g.read_raw(name, dst.data(), dst.size()*sizeof(float))) { - // A short read leaves dst half-overwritten. - dst = identity; - std::printf("vla(pi0): %s read failed - identity\n", name); + std::fprintf(stderr, "vla(pi0): %s read failed\n", name); + return false; } + return true; }; - read1d("state_mean", m.state_mean); - read1d("state_std", m.state_std); - read1d("action_mean", m.action_mean); - read1d("action_std", m.action_std); - return true; + bool ok = true; + ok &= read1d("state_mean", m.state_mean); + ok &= read1d("state_std", m.state_std); + ok &= read1d("action_mean", m.action_mean); + ok &= read1d("action_std", m.action_std); + return ok; } } diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index cbfc2d7..6a0079b 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -337,35 +337,40 @@ bool load_stats(gguf_reader & g, Pi05ModelArch & m) { m.state_std .assign(cfg.real_state_dim, 1.f); m.action_mean.assign(cfg.real_action_dim, 0.f); m.action_std .assign(cfg.real_action_dim, 1.f); + // Absent stats are a valid checkpoint: identity, carry on. Stats that are + // present but unreadable are not - falling back to identity there hands back + // un-denormalised actions with nothing in the log. Note stderr, not stdout: + // stdout is the action stream tests/predict_check.cpp diffs. auto read1d = [&](const char * name, std::vector & dst) { const ggml_tensor * t = g.meta(name); if (!t) { - std::printf("vla(pi05): %s missing - identity\n", name); - return; + std::fprintf(stderr, "vla(pi05): %s missing - identity\n", name); + return true; } if (t->ne[0] != (int64_t) dst.size()) { - std::printf("vla(pi05): %s dim mismatch - identity\n", name); - return; + std::fprintf(stderr, "vla(pi05): %s is %lld wide, expected %zu\n", + name, (long long) t->ne[0], dst.size()); + return false; } - const std::vector identity = dst; if (!g.read_raw(name, dst.data(), dst.size()*sizeof(float))) { - // A short read leaves dst half-overwritten. - dst = identity; - std::printf("vla(pi05): %s read failed - identity\n", name); + std::fprintf(stderr, "vla(pi05): %s read failed\n", name); + return false; } + return true; }; - read1d("state_mean", m.state_mean); - read1d("state_std", m.state_std); - read1d("action_mean", m.action_mean); - read1d("action_std", m.action_std); + bool ok = true; + ok &= read1d("state_mean", m.state_mean); + ok &= read1d("state_std", m.state_std); + ok &= read1d("action_mean", m.action_mean); + ok &= read1d("action_std", m.action_std); if (m.quantile_norm) { m.action_q01.assign(cfg.real_action_dim, -1.f); m.action_q99.assign(cfg.real_action_dim, 1.f); - read1d("action_q01", m.action_q01); - read1d("action_q99", m.action_q99); + ok &= read1d("action_q01", m.action_q01); + ok &= read1d("action_q99", m.action_q99); } - return true; + return ok; } } diff --git a/src/scratch_ctx.h b/src/scratch_ctx.h index 8db90ae..4e6a67e 100644 --- a/src/scratch_ctx.h +++ b/src/scratch_ctx.h @@ -41,12 +41,20 @@ class scratch_ctx { } ggml_context * reset(size_t arena) { + // Growing matters: an arena sized from the input shape would otherwise + // keep the first call's smaller pool and abort in ggml_new_tensor. + // Every call site passes a constant today. + if (ctx_ && arena > arena_) { + ggml_free(ctx_); + ctx_ = nullptr; + } if (ctx_) { ggml_reset(ctx_); return ctx_; } ggml_init_params p = { arena, nullptr, true }; - ctx_ = ggml_init(p); + ctx_ = ggml_init(p); + arena_ = arena; return ctx_; } @@ -70,6 +78,7 @@ class scratch_ctx { private: ggml_context * ctx_ = nullptr; ggml_gallocr_t galloc_ = nullptr; + size_t arena_ = 0; }; // Key is whatever shape the graph depends on (it needs operator==); IO holds the diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 607e79f..e3175f7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,9 +47,14 @@ target_link_libraries(test_config_guard PRIVATE vla_core) target_compile_options(test_config_guard PRIVATE -Wall -Wextra) add_test(NAME config_guard COMMAND test_config_guard) -# The A/B harness for the two BitVLA ternary-GEMM tilings (bitvla_gemm_check.cu) -# was never committed, so there is no target for it here. VLA_BITVLA_NARROW_GEMM=1 -# still selects the old one-tile-per-CTA kernel for a hand-run comparison. +# A/B harness for the two BitVLA ternary-GEMM tilings. Built so it cannot rot, +# not registered with ctest: it needs a GPU and is read by hand. +# VLA_BITVLA_NARROW_GEMM=1 selects the old one-tile-per-CTA kernel at runtime. +if(GGML_CUDA) + add_executable(bitvla_gemm_check bitvla_gemm_check.cu) + target_include_directories(bitvla_gemm_check PRIVATE ${CMAKE_SOURCE_DIR}/src) + target_link_libraries(bitvla_gemm_check PRIVATE bitvla_cuda_kernels) +endif() # Regression test for the in-tree BF16 CUDA kernels and the ggml hook they ride # on. Skips itself (exit 0) when no CUDA device is present. diff --git a/tests/bitvla_gemm_check.cu b/tests/bitvla_gemm_check.cu index 74a73b7..745af5d 100644 --- a/tests/bitvla_gemm_check.cu +++ b/tests/bitvla_gemm_check.cu @@ -26,7 +26,7 @@ * the LM and ViT actually use, plus a ragged M to exercise the row tail. */ -#include "../src/kernels/bitvla/bitnet_kernels.h" +#include "kernels/bitvla/bitnet_kernels.h" #include #include From c510716df6d0f731228afb22d4bfefef1cf55291 Mon Sep 17 00:00:00 2001 From: "An T. Le" Date: Tue, 1 Sep 2026 11:33:31 +0700 Subject: [PATCH 20/24] reconcile the ov.md fix counts and drop the superseded n1.7 paragraph --- docs/backend/ov.md | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 32c97d0..9613864 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -11,7 +11,7 @@ runtime on the configure line. > VLA-Adapter to about 3e-6. On the Arc B390 iGPU the speedup over the native CPU > backend runs from 3.0x to 9.6x. GR00T N1.7 translates and returns > plausible-looking actions that are simply wrong; it is the one open failure. -> Twelve fixes were needed, ten of them inside ggml's OpenVINO backend, which is +> Thirteen fixes were needed, eleven of them inside ggml's OpenVINO backend, which is > written against llama.cpp's graphs and had never seen a vision tower or an > action expert - see [What had to change](#what-had-to-change). > @@ -132,8 +132,12 @@ binaries: `libopenvino.so` and its TBB live under `/opt/intel`. Configure fails early with a pointer back here if the runtime is not on `CMAKE_PREFIX_PATH`. `scripts/patch_ggml_openvino.py` runs as the FetchContent patch step, so the six -ggml fixes described in its docstring are applied automatically and re-applied on -a clean reconfigure. There is no manual `git apply`. +ggml fixes described in its docstring are applied automatically. There is no +manual `git apply`. The step only runs when FetchContent populates the source +dir, so a `build/_deps` left over from an older checkout keeps the hunks it was +patched with: delete it after pulling rather than trusting a reconfigure. The +script checks each hunk on its own and fails loudly on a tree it cannot bring up +to date. ## Run @@ -148,7 +152,7 @@ Two lines identify the selection at startup: ```text OpenVINO: using device GPU -vla: backend = OPENVINO (requested device GPU) +vla: backend = OPENVINO (asked for GPU, see ggml's "using device" line) ``` The first comes from ggml and is authoritative: an unavailable device logs a @@ -165,7 +169,8 @@ timeout well above the first request. Do **not** set `GGML_OPENVINO_CACHE_DIR` to carry them across restarts. It produces silently wrong actions here - see -[Known issues](#known-issues). The backend warns at startup if it is set. +[Known issues](#known-issues). `backend_init` clears it and says so; +`VLA_ALLOW_OV_CACHE=1` keeps it if you have verified the outputs yourself. ## Results @@ -287,7 +292,7 @@ larger through a model builder that assumes a decoder-only LLM. The literal path is the one that fits a vision tower and an action expert. An explicit setting still wins. -The other ten are in ggml's OpenVINO backend itself, applied by +The other eleven are in ggml's OpenVINO backend itself, applied by `scripts/patch_ggml_openvino.py` at configure time. Its docstring carries the detail; in short each narrows an llama.cpp-shaped assumption that is stricter than the ggml contract, or fills a gap: @@ -302,7 +307,10 @@ than the ggml contract, or fills a gap: | Folded weights padded to full rank | a 2-D weight becomes a rank-2 constant, but views index it at ggml rank | | CONCAT input ranks aligned | same rank-2 constants, and concat cannot broadcast rank | | Missing op translators | RELU, GELU_ERF, NEG, SQR had no table entry | -| Naive-path graph cache | that path re-compiled the whole model on every graph_compute | +| Naive-path graph cache | that path re-compiled the whole model on every graph_compute, and its `graph_key` is a node count plus two names, which two graphs can share | +| Interleaved-mrope sectors bounded | the sector cycle ignored `sections`, so the last few took the wrong stream | +| Interleaved-mrope mode passed through | the shared sin/cos table was built with the plain-rope layout | +| Naive-path threshold settable | the 20-node constant is what picks the literal path | Four are worth expanding. @@ -337,8 +345,8 @@ as a rank-2 constant, which is what a GEMM operand wants, but the graph indexes it at full ggml rank. Evo-1 views Q, K and V out of one fused `attn_in` weight and got `Axis 2 out of the tensor rank range [-2, 1]`. Padding in `process_view_input_new` fixed that class generally, and padding in the concat -translator let vla.cpp **delete** an arch-specific workaround that had moved -SmolVLA's time tiles into their own buffer. +translator is what saves each arch from working around it - SmolVLA would +otherwise need its time tiles moved into a buffer of their own. **The naive-path cache** is about speed, not correctness. The dynamic and static paths keep a `graph_key`-indexed cache; the naive path had none, so it rebuilt @@ -363,10 +371,11 @@ GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_CACHE_DIR=$dir # warm: max |delta| 2.9e ``` Nothing is logged - the actions are simply wrong, which for a policy server is -the worst possible failure mode. `backend_init` warns at startup when the -variable is set. Unverified guess at the cause: the blob key does not capture -something that differs between vla.cpp's several graphs, so one graph gets -another's blob. In practice, pay the compile once per process and leave it unset. +the worst possible failure mode. `backend_init` therefore clears the variable and +says so; set `VLA_ALLOW_OV_CACHE=1` alongside it to keep it. Unverified guess at +the cause: the blob key does not capture something that differs between +vla.cpp's several graphs, so one graph gets another's blob - the same class of +bug as the in-process `graph_key` above, which is now keyed on shapes. In practice, pay the compile once per process and leave it unset. **The NPU takes two of the eight archs, and fails four different ways.** SmolVLA and π0.5 run. The others do not: @@ -410,17 +419,6 @@ is therefore **untested**: only BitVLA emits it, and BitVLA never reaches this backend. It is in the table because it is a real gap in ggml-openvino, not because anything here exercises it. -**GR00T N1.7 translates but computes the wrong answer.** It runs to completion -and returns a full action chunk, so nothing errors, but 86% of the 5,280 values -are off by more than 1e-2 and the worst is 1.48 against a peak of 0.948. The -result is deterministic and identical on the CPU and GPU plugins, so this is a -translation bug rather than a device or precision effect. It is **not** the mrope -handling: fixes 9 and 10 in `scripts/patch_ggml_openvino.py` both target that -path and neither moved the number by a digit. VLA-JEPA shares the same Qwen3-VL -vision tower and the same interleaved mrope and is only ~1% off, which points at -GR00T N1.7's own action expert rather than anything shared. Not yet diagnosed; -the arch stays `-` in the README matrix. - **GR00T N1.7 is the one open failure.** It translates, returns a full action chunk, and the values are wrong: max|delta| 1.477, rms 8.3e-2 against a peak of 0.948, with 86% of 5280 values off by more than 1e-2. Deterministic, and From 7f99982a4cc0e0aaece91a4d6c581be750270c60 Mon Sep 17 00:00:00 2001 From: "An T. Le" Date: Tue, 1 Sep 2026 11:43:37 +0700 Subject: [PATCH 21/24] read the llama tag from one place so ci stops guessing at it --- .github/workflows/build.yml | 5 +---- scripts/llama_tag.sh | 32 ++++++++++++++++++++++++++++++++ scripts/print_versions.sh | 3 +-- 3 files changed, 34 insertions(+), 6 deletions(-) create mode 100755 scripts/llama_tag.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 624768c..6a96ce9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,10 +52,7 @@ jobs: # silently reuses the wrong _deps. - name: read llama.cpp pin id: pin - run: | - tag=$(grep -m1 'set(VLA_LLAMA_TAG' CMakeLists.txt | grep -oE 'b[0-9]+') - test -n "$tag" - echo "tag=$tag" >> "$GITHUB_OUTPUT" + run: echo "tag=$(bash scripts/llama_tag.sh)" >> "$GITHUB_OUTPUT" - uses: actions/cache@v6 with: path: build/_deps diff --git a/scripts/llama_tag.sh b/scripts/llama_tag.sh new file mode 100755 index 0000000..53afadb --- /dev/null +++ b/scripts/llama_tag.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Print the llama.cpp tag CMakeLists.txt pins. Two consumers read it - the CI +# cache key and print_versions.sh - and both used to grep for it themselves. +# That grep broke silently once when the tag moved behind ${VLA_LLAMA_TAG} and +# loudly again when the default moved behind another variable, so it lives here. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CMAKE="${1:-${ROOT}/CMakeLists.txt}" + +tag="$(grep -m1 -oE 'set\(_vla_llama_tag_default "[^"]+"' "${CMAKE}" | grep -oE '"[^"]+"' | tr -d '"' || true)" + +if [[ -z "${tag}" ]]; then + echo "llama_tag: no _vla_llama_tag_default in ${CMAKE}" >&2 + exit 1 +fi +echo "${tag}" diff --git a/scripts/print_versions.sh b/scripts/print_versions.sh index 4bc992b..678dfaa 100644 --- a/scripts/print_versions.sh +++ b/scripts/print_versions.sh @@ -84,8 +84,7 @@ echo LLAMA_HEAD=$(git_head_or_dash "$LLAMA_DIR") LLAMA_DESC=$(git_describe_or_dash "$LLAMA_DIR") echo "- HEAD: \`${LLAMA_HEAD}\` (\`${LLAMA_DESC}\`)" -# GIT_TAG is ${VLA_LLAMA_TAG}, so read the cache variable, not the fetch call. -LLAMA_TAG=$(grep -m1 'set(VLA_LLAMA_TAG' "$ROOT/CMakeLists.txt" | grep -oE 'b[0-9]+' || echo '?') +LLAMA_TAG=$("$ROOT/scripts/llama_tag.sh" 2>/dev/null || echo '?') echo "- expected pinned tag (from \`CMakeLists.txt\`): \`${LLAMA_TAG}\`" # ---- GGUFs ---- From 7c2c89e0687878affc1eb2e88a2c049e71c2627d Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Tue, 1 Sep 2026 12:20:21 +0700 Subject: [PATCH 22/24] require a rope before permute op_case 2, which fixes gr00t n1.7 --- README.md | 2 +- docs/backend/ov.md | 95 ++++++++++++++-------------------- scripts/patch_ggml_openvino.py | 48 +++++++++++++++-- 3 files changed, 83 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 0c54cad..7c58da1 100644 --- a/README.md +++ b/README.md @@ -319,7 +319,7 @@ supported (released and benchmarked), `~` = in progress, `-` = planned. | [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | | [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | Y | | [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | -| [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | - | +| [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | Y | | [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | | [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | | [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 9613864..4cb5583 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -5,15 +5,14 @@ account of how far it currently runs. Like SYCL, OpenVINO is **not** auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. -> **Status: seven architectures translate faithfully, one is wrong.** SmolVLA, -> π0.5, Evo-1, VLA-Adapter, GR00T N1.5, GR00T N1.6 and VLA-JEPA all agree with a -> CPU-backend reference to 1.3e-3 or better on the OpenVINO CPU plugin - Evo-1 and -> VLA-Adapter to about 3e-6. On the Arc B390 iGPU the speedup over the native CPU -> backend runs from 3.0x to 9.6x. GR00T N1.7 translates and returns -> plausible-looking actions that are simply wrong; it is the one open failure. -> Thirteen fixes were needed, eleven of them inside ggml's OpenVINO backend, which is -> written against llama.cpp's graphs and had never seen a vision tower or an -> action expert - see [What had to change](#what-had-to-change). +> **Status: all eight tested architectures translate faithfully.** SmolVLA, π0.5, +> Evo-1, VLA-Adapter, GR00T N1.5, GR00T N1.6, GR00T N1.7 and VLA-JEPA all agree +> with a CPU-backend reference to 1.3e-3 or better on the OpenVINO CPU plugin - +> Evo-1 and VLA-Adapter to about 3e-6. On the Arc B390 iGPU the speedup over the +> native CPU backend runs from 3.0x to 9.6x. Fifteen fixes were needed, thirteen +> of them inside ggml's OpenVINO backend, which is written against llama.cpp's +> graphs and had never seen a vision tower or an action expert - see +> [What had to change](#what-had-to-change). > > Note the baseline: OpenVINO executes the checkpoint's BF16 weights at F32, so > compare against `--weight-dtype f32` or you will charge the backend for a @@ -188,10 +187,7 @@ reason in [Known issues](#known-issues). | SmolVLA | 512 | 1,364 ms | 1,340 ms | **451 ms** (3.0x) | 1,162 ms | | Evo-1 | 448 | 3,114 ms | 4,523 ms | **563 ms** (5.5x) | not supported | | π0.5 | 224 | 2,802 ms | 4,285 ms | **683 ms** (4.1x) | 916 ms | -| GR00T N1.7 | 256 | not timed | not timed | not timed | not attempted | - -GR00T N1.7 is not timed because it computes the wrong answer - a latency for work -that is not the same work would be misleading. +| GR00T N1.7 | 256 | 1,146 ms | not timed | **288 ms** (4.0x) | not attempted | The iGPU is the reason to use this backend, and it pays off most where the model is most vision-heavy. The OpenVINO CPU plugin is at best parity with ggml's own @@ -224,7 +220,7 @@ both and take the smaller as the fidelity figure: | GR00T N1.5 | 5.5e-3 | **6.0e-4** | F32 | | GR00T N1.6 | 5.0e-3 | **1.0e-3** | F32 | | SmolVLA | **8.9e-4** | 1.3e-3 | BF16 | -| GR00T N1.7 | 1.5e0 | 1.5e0 | neither - it is wrong | +| GR00T N1.7 | 1.5e0 | **4.2e-4** | F32 | Evo-1 and VLA-Adapter agree with an F32 reference to six decimal places, which is as close to "the translation is exact" as this harness can show - for those two, @@ -249,9 +245,9 @@ Against the F32 reference, which is the fidelity number: | GR00T N1.5 | 6.0e-4 | 4.7e-3 | plugin throws | | GR00T N1.6 | 1.0e-3 | 2.0e-3 | plugin throws | | SmolVLA | 8.9e-4 | 1.3e-3 | 1.7e-2 | -| GR00T N1.7 | 1.5e0 | 1.5e0 | not attempted | +| GR00T N1.7 | 4.2e-4 | 4.1e-3 | not attempted | -Every arch except GR00T N1.7 is inside the 2.9e-3 bar on the CPU plugin, most by +Every tested arch is inside the 2.9e-3 bar on the CPU plugin, most by one to three orders of magnitude. On the GPU the picture is looser because that plugin computes in F16: VLA-JEPA (8.7e-3), GR00T N1.5 (4.7e-3) and VLA-Adapter (3.2e-3) sit outside the bar there even though all three are far inside it on the CPU plugin. Judge translation @@ -262,8 +258,6 @@ one. The other is SmolVLA on the NPU (1.7e-2), whose compile config turns on dynamic quantization - π0.5 on the same device stays at 9.9e-4, so that is a property of the model on that device rather than of the backend. -GR00T N1.7 is wrong outright - see [What is left](#what-is-left). - ## What had to change Two fixes on the vla.cpp side. Both are ordinary correctness fixes that happen to @@ -292,13 +286,14 @@ larger through a model builder that assumes a decoder-only LLM. The literal path is the one that fits a vision tower and an action expert. An explicit setting still wins. -The other eleven are in ggml's OpenVINO backend itself, applied by +The other thirteen are in ggml's OpenVINO backend itself, applied by `scripts/patch_ggml_openvino.py` at configure time. Its docstring carries the detail; in short each narrows an llama.cpp-shaped assumption that is stricter than the ggml contract, or fills a gap: | Fix | What it addresses | |---|---| +| **PERMUTE op_case 2 requires a ROPE** | **assumes any permute of a view is a rope'd query** | | **GELU translated as tanh, not erf** | **assumes ggml's GELU is the exact erf form** | | Intel OpenCL platform selection | assumes the first OpenCL platform is Intel's | | RESHAPE `op_case` guard | assumes a reshape flattening dims 0-2 is the KV-cache flatten | @@ -312,7 +307,12 @@ than the ggml contract, or fills a gap: | Interleaved-mrope mode passed through | the shared sin/cos table was built with the plain-rope layout | | Naive-path threshold settable | the 20-node constant is what picks the literal path | -Four are worth expanding. +Five are worth expanding. + +**The PERMUTE op_case guard** is what makes GR00T N1.7 correct, and it is the +subtlest of the set: a classifier that sent an ordinary head-split permute down +an LLM-specific rewrite. See [What is left](#what-is-left) for the bisect that +found it. **The GELU mode** is the highest-yield single fix in the list. ggml's `GGML_UNARY_OP_GELU` is the *tanh* approximation - its CPU kernel additionally @@ -387,7 +387,7 @@ and π0.5 run. The others do not: | VLA-JEPA | compiles and runs, returns all `NaN` | | GR00T N1.5 | NPUW partitioning throws (`partitioning.cpp:1350`) | | GR00T N1.6 | plugin throws (`core.cpp:117`) | -| GR00T N1.7 | not attempted - wrong on CPU and GPU already | +| GR00T N1.7 | not attempted | Five archs, four distinct failures, none of them vla.cpp's. The two alignment rejections are Intel's NPU compiler: 1025 is Evo-1's 1024 @@ -419,41 +419,24 @@ is therefore **untested**: only BitVLA emits it, and BitVLA never reaches this backend. It is in the table because it is a real gap in ggml-openvino, not because anything here exercises it. -**GR00T N1.7 is the one open failure.** It translates, returns a full action -chunk, and the values are wrong: max|delta| 1.477, rms 8.3e-2 against a peak of -0.948, with 86% of 5280 values off by more than 1e-2. Deterministic, and -identical on the CPU and GPU plugins. Ruled out so far, each with evidence: - -- *Precision.* Identical against BF16 and F32 references. OpenVINO is 1530x less - weight-dtype-sensitive than ggml CPU here, and the model's own bf16/f32 - sensitivity is rms 7.2e-4 against the error's 8.3e-2. -- *Translation-path selection.* The naive path is taken by default and - `is_model_splitted` returns false for every graph of this arch; forcing all - graphs down the decoder-only-LLM path instead changes the answer by 1e-5 while - both remain 1.477 from the reference. -- *Sequence length and token composition.* Swept 3.7x (SEQ 70 to 262) by two - independent routes; relative error stayed within 0.2545-0.2841 and the fraction - off by >1e-2 within 84.7-86.6%. Nothing accumulates. -- *Matmul precision and the compiled-model cache.* Both bitwise no-ops. -- *The GELU mode*, which fixed VLA-JEPA and GR00T N1.5, moves N1.7 by nothing. -- *A missing or unsupported op.* GR00T N1.6 (works) and N1.7 (broken) have the - same 17-op vocabulary, and LM layer 0 is node-for-node identical except that - N1.7's position input is `[4*SEQ]` for IMROPE where N1.6's is `[SEQ]` for NEOX. - VLA-JEPA also uses IMROPE and is now clean, which weakens that lead. - -Three earlier claims about N1.7 turned out to be **measurement artifacts** and -should not be reused. That its first LM block is 73-95% wrong: OpenVINO's -`lm_h_00..03` dumps are bit-exact copies of the *input* arrays and `lm_h_04..15` -are zeros, because the main graph has exactly one real output, `action_pred`. -That `--flash-attn 1` yields 0.948: it actually fails with "Got less inputs than -expected" and returns no actions, and 0.948 is `max|reference|` - the number you -get comparing against nothing. And that honouring `GGML_TENSOR_FLAG_OUTPUT` -changes the answer: same artifact. - -The next step follows from the first artifact. The stage dump cannot see inside -the graph because ggml-openvino writes back only true graph outputs, so either -add a debug mode that materialises selected intermediates as `ov::Result`s, or -bisect with cut-down graphs. +**GR00T N1.7 was the last failure and is fixed.** It used to return +plausible-looking but wrong actions - max|delta| 1.477, 86% of values off by more +than 1e-2. Bisecting with cut-down graphs found it: truncating the graph at a +stage makes that stage the terminal node, which is the only way to observe an +interior tensor under this backend. Everything through the vision tower, the LM +and the vlsa stack was clean at 0.05-0.08%, and the error appeared entirely +inside the DiT action expert - specifically its cross-attention `V`, 139% wrong +while `K` from the same call was 0.04%. + +The cause was `compute_op_case`: PERMUTE op_case 2 rewrites a tensor as +`[n_seq, -1, n_heads, head_size]` before transposing, which is right for +llama.cpp's rope'd query and nothing else, but it was reached by *any* permute +whose source is a view of a non-leaf. GR00T N1.7's `ggml_permute(view, 1,2,0,3)` +over a fused KV projection took that path and came out with its elements +rearranged. `K` uses `permute(0,2,1,3)` and happened to survive the same rewrite, +which is why only `V` broke. Requiring an actual ROPE at the end of the +view/reshape/cont chain sends every other permute to op_case 1, the plain +transpose: 27% -> 0.005%, with every other arch bit-identical. **Untested archs.** π0 and OpenVLA-OFT are untested here for want of a local checkpoint, not because anything is known to block them; both use op sets already diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index e46201f..c37aa53 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Eleven fixes to the fetched ggml OpenVINO backend. +"""Thirteen fixes to the fetched ggml OpenVINO backend. ggml-openvino is written against llama.cpp's graphs: one decoder-only transformer, one position input, an F16 KV cache. vla.cpp drives it with vision @@ -134,10 +134,29 @@ atoi turns junk into 0 and that would send every graph down the LLM builder with nothing said. -Idempotent per hunk, not per file: a tree patched by an older checkout is -missing the hunks added since, and a file-wide marker would skip them and leave -a build that runs and is quietly wrong. Nothing is written until every anchor -has matched, so a mismatch leaves the tree untouched. + 12. openvino/op_table.cpp - translate GELU as the tanh approximation. + ggml's GGML_UNARY_OP_GELU is the *tanh* approximation (its CPU kernel also + reads an fp16 lookup table), but ov::op::v7::Gelu defaults to the exact erf + form and both ggml GELU ops were mapped onto that default. Per node the + difference is small; a Qwen3-VL vision tower has dozens of them and it + compounds. Setting the mode explicitly moved VLA-JEPA from 5.5e-3 to 1.1e-4 + and GR00T N1.5 from 2.7e-2 to 6.0e-4, turning both from "runs but drifts" + into supported. The highest-yield single fix here. + + 13. ggml-decoder.cpp - require a ROPE before taking PERMUTE op_case 2. + op_case 2 rewrites the tensor as [n_seq, -1, n_heads, head_size] and only + then transposes, which is correct for llama.cpp's rope'd query and nothing + else. The classifier reached it for ANY permute whose source is a view of a + non-leaf, so GR00T N1.7's DiT cross-attention V - ggml_permute(view, 1,2,0,3) + over a fused KV projection - was reshaped into a shape unrelated to it and + came out with its elements rearranged: V was 139% wrong while K, which uses + permute(0,2,1,3) and happened to survive the same rewrite, was 0.04%. That + turned into a 27% error in the final actions. Walking the view/reshape/cont + chain and requiring a ROPE at the end sends every other permute to op_case 1, + the plain transpose. This is what makes GR00T N1.7 correct (27% -> 0.005%). + +Idempotent - re-running on a patched tree is a no-op, so a reconfigure that +re-populates the FetchContent source dir is safe either way. Usage: scripts/patch_ggml_openvino.py [] """ @@ -506,6 +525,25 @@ ), ], "ggml/src/ggml-openvino/ggml-decoder.cpp": [ + ( + """ } else { + // rope'ed query tensor + op_case = 2;""", + """ } else { + // vla.cpp: op_case 2 rewrites the tensor as [n_seq, -1, n_heads, + // head_size] before transposing, which is only correct for + // llama.cpp's rope'd query. It was reached by ANY permute whose + // source is a view of a non-leaf, so a DiT head split -- GR00T + // N1.7's cross-attention V, ggml_permute(view, 1,2,0,3) -- was + // reshaped into a shape that has nothing to do with it and came out + // with its elements rearranged. Require the rope. + const ggml_tensor * prod = node->src[0]; + while (prod && prod->src[0] && + (prod->op == GGML_OP_VIEW || prod->op == GGML_OP_RESHAPE || prod->op == GGML_OP_CONT)) { + prod = prod->src[0]; + } + op_case = (prod && prod->op == GGML_OP_ROPE) ? 2 : 1;""", + ), ( """ } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { op_case = 3;""", From ca237387dda874f69ed1d110588f25afb48a5e95 Mon Sep 17 00:00:00 2001 From: Khanh Dang Nguyen Date: Tue, 1 Sep 2026 17:06:45 +0700 Subject: [PATCH 23/24] fix the openvino regressions from the b10729 bump: dead position-input hunk, gpu gemm post-ops, pi0 precision --- CHANGELOG.md | 42 ++++++- README.md | 5 +- docs/UPSTREAMING.md | 6 +- docs/backend/ov.md | 161 +++++++++++++++++---------- scripts/patch_ggml_openvino.py | 193 ++++++++++++++++++++++++++------- scripts/upstream_split.py | 106 ++++++++++++++---- src/backend.h | 16 +++ 7 files changed, 403 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e1e1fe..1fb5f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,14 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke - **OpenVINO backend.** `-DGGML_OPENVINO=ON` runs the archs on Intel CPUs, iGPUs and NPUs through ggml's OpenVINO backend. SmolVLA, π0.5, Evo-1 and VLA-Adapter - match an F32 CPU reference to 1e-3; on an Arc B390 iGPU that is 3.1x to 8.2x - the native CPU backend. GR00T N1.5/N1.6 and VLA-JEPA run but drift, GR00T N1.7 - is wrong, π0 and OpenVLA-OFT are untested. See `docs/backend/ov.md`. + match an F32 CPU reference to 1e-3; on an Arc B390 iGPU that is 3.0x to 9.6x + the native CPU backend. All nine tested archs are inside the accuracy bar on the + OpenVINO CPU plugin and on the iGPU. OpenVLA-OFT is untested. + See `docs/backend/ov.md`. - `scripts/install_ov.sh` installs the OpenVINO runtime and the Intel GPU/NPU driver stack on Ubuntu 22.04 and 24.04, with the runtime archive checksummed against a digest pinned in the script. -- `scripts/patch_ggml_openvino.py` applies eleven fixes to the fetched +- `scripts/patch_ggml_openvino.py` applies thirteen fixes to the fetched ggml-openvino sources at configure time. Each hunk is checked on its own, so a `build/_deps` patched by an older checkout fails loudly instead of building something quietly wrong. @@ -22,13 +23,44 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke - CI now checks that both llama.cpp patch scripts still apply, on a copy of the fetched tree. Neither ran on a CPU build, so their anchors could rot unnoticed until someone configured a CUDA or OpenVINO tree. -- `docs/UPSTREAMING.md` and `scripts/upstream_split.py` regroup the eleven +- `docs/UPSTREAMING.md` and `scripts/upstream_split.py` regroup the thirteen ggml-openvino fixes into one llama.cpp branch per PR. They are generic backend defects, not vla.cpp workarounds; landing them upstream removes the configure-time patch step entirely. ### Fixed +- Two elementwise adds stacked on a GEMM came out wrong on the Intel iGPU. The + GPU plugin folds elementwise ops into the preceding GEMM as post-ops, and given + `ADD(ADD(residual, GEMM), graph_input)` it folds both and silently drops the + second operand - the result equals the inner add. A llama.cpp graph never builds + that chain; a VLA does, wherever a vision tower's features are added on top of an + FFN residual. VLA-JEPA (5.4e-1) and GR00T N1.7 (1.9e0) were wrong on the iGPU + while matching the CPU plugin to 1e-4. Re-associating the two adds so the GEMM + keeps one post-op puts both at 2.6e-3. Bisected with `GGML_OPENVINO_DEBUG_NODE`. +- π0's action dims drifted 4e-2 on the iGPU and its gripper flipped a step late, + because the GPU plugin computes in F16 and π0 unrolls its whole denoise loop + inside one graph. `GGML_OPENVINO_GPU_PRECISION` now exposes the plugin's + inference precision; `backend_init` defaults it to f32 for π0 alone, which costs + about 3x on that arch and puts it at 6.5e-5. +- `scripts/patch_ggml_openvino.py` now fails if `EDITS` has a duplicate key. Python + keeps the last one silently, and a duplicate briefly removed the whole Intel + OpenCL platform fix from the patch without any error. +- The position-input fix stopped running when llama.cpp moved to `b10729`. That + release relocated the naming out of `GgmlOvDecoder::get_graph_input_ov_name()`, + which the patch guards, into a new free `get_tensor_graph_input_ov_name()`, and + left the member behind with no callers. The hunk still applied cleanly, so + nothing failed loudly - SmolVLA and π0.5 simply stopped returning actions + ("Argument shapes are inconsistent", a 113-token prefix ROPE reading the + 50-token suffix's table). Both functions are guarded now, and the patch script + says to check for a live caller, not just a matching anchor, on every tag bump. +- `scripts/upstream_split.py` addressed hunks by position in the patch script's + edit list. Adding a hunk to the front of a file's list silently handed every + later hunk to the wrong branch, and its own coverage count still read 29/29 + because each index was still used exactly once. Two branches had been swapped + this way. Hunks are now addressed by a unique substring of their anchor, which + fails loudly instead. The PERMUTE `op_case` fix, which had no branch at all, + now has one. - `graph_unique_names` renamed through `ggml_format_name`, which passes the tensor's own name to `vsnprintf` as both destination and `%s` source. glibc empties it, so every duplicate node became the bare string `#`. diff --git a/README.md b/README.md index 7c58da1..3574c9e 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ supported (released and benchmarked), `~` = in progress, `-` = planned. | Model | CPU (x86-64 / ARM) | CUDA | [SYCL (Intel)](docs/backend/sycl.md) | [Metal](docs/backend/metal.md) | [OpenVINO](docs/backend/ov.md) | |---|:--:|:--:|:--:|:--:|:--:| | [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | -| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | ~ | +| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | Y | | [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | | [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | Y | | [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | @@ -326,6 +326,9 @@ supported (released and benchmarked), `~` = in progress, `-` = planned. | [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | ~ | | [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | +Under OpenVINO, π0 runs the Intel iGPU at F32 rather than the default F16, which +`backend_init` sets for it - see [Known issues](docs/backend/ov.md#known-issues). + --- ## Contributing diff --git a/docs/UPSTREAMING.md b/docs/UPSTREAMING.md index 3f31c7c..842d4f1 100644 --- a/docs/UPSTREAMING.md +++ b/docs/UPSTREAMING.md @@ -37,7 +37,9 @@ python3 scripts/upstream_split.py | `openvino-gelu-modes` | ggml's tanh `GELU` is mapped onto ov's erf default, and `GELU_ERF` has no entry at all | **high**: wrong activation on every GELU node. Fixing it moved GR00T N1.5 and VLA-JEPA inside the accuracy bar | | `openvino-intel-opencl-platform` | the GPU remote context takes the first OpenCL platform | low, but a hard startup abort when it bites | | `openvino-imrope-sections` | the IMROPE sector cycle ignores `sections` | low, no measured output change | -| `openvino-imrope-mode` | the shared sin/cos table is built without the imrope flag | low, untested path | +| `openvino-gemm-double-eltwise` | the GPU plugin folds two chained elementwise adds into the preceding GEMM and silently drops the second operand | **highest**: wrong output on GPU with nothing logged, on any graph that adds a tower's features on top of an FFN residual | +| `openvino-gpu-precision-env` | the GPU plugin's f16 default compounds through a long serial chain in one graph | medium: exposes `GGML_OPENVINO_GPU_PRECISION`, default unchanged | +| `openvino-permute-op-case` | PERMUTE `op_case` 2 is reached by any permute over a view of a non-leaf, not just llama.cpp's rope'd query | **high**: a DiT cross-attention V comes out with its elements rearranged and nothing is reported | `RELU`, `NEG` and `SQR` were in the patch too. Upstream added all three in `b10729`, so they are not in the series. @@ -76,7 +78,7 @@ already proven on hardware. ## Status -The eleven branches carry code that has run on an Intel Core Ultra X7 358H (Arc +The thirteen branches carry code that has run on an Intel Core Ultra X7 358H (Arc B390 iGPU, AI Boost NPU) through vla.cpp's own OpenVINO builds - see `docs/backend/ov.md` for what that covered. They have **not** been compiled from these branches: no OpenVINO runtime is installed on the machine that split them, diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 4cb5583..697ba90 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -5,24 +5,28 @@ account of how far it currently runs. Like SYCL, OpenVINO is **not** auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO runtime on the configure line. -> **Status: all eight tested architectures translate faithfully.** SmolVLA, π0.5, -> Evo-1, VLA-Adapter, GR00T N1.5, GR00T N1.6, GR00T N1.7 and VLA-JEPA all agree -> with a CPU-backend reference to 1.3e-3 or better on the OpenVINO CPU plugin - -> Evo-1 and VLA-Adapter to about 3e-6. On the Arc B390 iGPU the speedup over the -> native CPU backend runs from 3.0x to 9.6x. Fifteen fixes were needed, thirteen -> of them inside ggml's OpenVINO backend, which is written against llama.cpp's -> graphs and had never seen a vision tower or an action expert - see -> [What had to change](#what-had-to-change). +> **Status: all nine tested architectures translate faithfully, on the CPU plugin +> and on the iGPU.** SmolVLA, π0, π0.5, Evo-1, VLA-Adapter, GR00T N1.5, GR00T N1.6, +> GR00T N1.7 and VLA-JEPA all agree with a CPU-backend reference to 1.4e-3 or better +> on the OpenVINO CPU plugin - Evo-1 and VLA-Adapter to about 3e-6 - and eight of +> the nine are inside the same bar on the Arc B390 iGPU, where the speedup over the +> native CPU backend runs from 3.0x to 9.6x. VLA-Adapter is the one outside it +> there, at 5.3e-3 on actions peaking at 0.62, which is the plugin's F16 arithmetic +> rather than a translation error. +> +> Fifteen fixes were needed, thirteen of them inside ggml's OpenVINO backend, which +> is written against llama.cpp's graphs and had never seen a vision tower or an +> action expert - see [What had to change](#what-had-to-change). > > Note the baseline: OpenVINO executes the checkpoint's BF16 weights at F32, so > compare against `--weight-dtype f32` or you will charge the backend for a > precision upgrade. See [Picking the right baseline](#picking-the-right-baseline). Measured on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 -iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10331`, -on the checkpoints under `vrfai/` on the Hub. `CMakeLists.txt` has since moved to -`b10729`, which is byte-identical on the CPU backend for all eleven archs; the -OpenVINO numbers below have not been re-measured on it. +iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10729`, +on the checkpoints under `vrfai/` on the Hub. The **fidelity** numbers for the CPU +and GPU plugins were all re-measured on that pin. The **latency** table and the +NPU column still date from `b10331` and are marked where they appear. OpenVINO is Intel's inference toolkit; ggml's backend translates a ggml compute graph into an OpenVINO model and hands it to the CPU, GPU or NPU plugin, which @@ -130,8 +134,8 @@ cmake --build build-ov -j$(nproc) binaries: `libopenvino.so` and its TBB live under `/opt/intel`. Configure fails early with a pointer back here if the runtime is not on `CMAKE_PREFIX_PATH`. -`scripts/patch_ggml_openvino.py` runs as the FetchContent patch step, so the six -ggml fixes described in its docstring are applied automatically. There is no +`scripts/patch_ggml_openvino.py` runs as the FetchContent patch step, so the +thirteen ggml fixes described in its docstring are applied automatically. There is no manual `git apply`. The step only runs when FetchContent populates the source dir, so a `build/_deps` left over from an older checkout keeps the hunks it was patched with: delete it after pulling rather than trusting a reconfigure. The @@ -178,6 +182,11 @@ one camera view, best of 4-6 iterations after 3 warmups. "CPU backend" is ggml's own CPU backend on the same 16-core host. No `GGML_OPENVINO_CACHE_DIR`, for the reason in [Known issues](#known-issues). +These latencies were taken at llama.cpp `b10331` and have not been re-timed on +`b10729`; the fidelity numbers under [Full results](#full-results) have. Read the +GPU column for VLA-JEPA and GR00T N1.7 as the cost of a wrong answer - see +[Known issues](#known-issues). + | Model | input | CPU backend | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | |---|---|---:|---:|---:|---:| | VLA-JEPA | 256 | 1,046 ms | 1,265 ms | **127 ms** (8.2x) | returns NaN | @@ -236,22 +245,32 @@ precision is the same size as the numbers being reported. Against the F32 reference, which is the fidelity number: -| Model | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | +| Model | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU (`b10331`) | |---|---:|---:|---:| -| VLA-Adapter | 2.4e-6 | 3.2e-3 | not supported | -| Evo-1 | 3.5e-6 | 1.2e-3 | not supported | -| π0.5 | 3.5e-5 | 4.7e-4 | 9.9e-4 | -| VLA-JEPA | 1.1e-4 | 8.7e-3 | returns NaN | -| GR00T N1.5 | 6.0e-4 | 4.7e-3 | plugin throws | -| GR00T N1.6 | 1.0e-3 | 2.0e-3 | plugin throws | -| SmolVLA | 8.9e-4 | 1.3e-3 | 1.7e-2 | -| GR00T N1.7 | 4.2e-4 | 4.1e-3 | not attempted | - -Every tested arch is inside the 2.9e-3 bar on the CPU plugin, most by -one to three orders of magnitude. On the GPU the picture is looser because that -plugin computes in F16: VLA-JEPA (8.7e-3), GR00T N1.5 (4.7e-3) and VLA-Adapter (3.2e-3) sit outside the bar there -even though all three are far inside it on the CPU plugin. Judge translation -fidelity on the CPU plugin; treat the GPU as a separate precision target. +| Evo-1 | 2.2e-6 | 6.0e-4 | not supported | +| VLA-Adapter | 3.9e-6 | 6.8e-3 | not supported | +| π0.5 | 6.1e-5 | 7.6e-4 | 9.9e-4 | +| VLA-JEPA | 7.7e-5 | 2.6e-3 | returns NaN | +| GR00T N1.7 | 3.9e-4 | 2.6e-3 | not attempted | +| π0 | 5.8e-4 | 6.6e-5 | not attempted | +| GR00T N1.5 | 6.0e-4 | 4.6e-3 | plugin throws | +| GR00T N1.6 | 1.1e-3 | 1.4e-3 | plugin throws | +| SmolVLA | 1.4e-3 | 2.6e-3 | 1.7e-2 | + +Every tested arch is inside the 2.9e-3 bar on the CPU plugin, most by one to +three orders of magnitude, and eight of the nine are inside it on the GPU as well +(GR00T N1.5's 4.6e-3 against F32 is 1.6e-3 against BF16, the tighter reference for +that arch). + +The one that sits outside is **VLA-Adapter**, at 6.8e-3 against F32 and 5.3e-3 +against BF16 on actions peaking at 0.62. That is the GPU plugin computing in F16 +and is a precision effect, not a translation error: the same arch is 3.9e-6 on the +CPU plugin. Judge translation fidelity on the CPU plugin and treat the GPU as a +separate precision target. + +π0 is the exception in the other direction - it is *tighter* on the GPU (6.6e-5) +than on the CPU plugin, because it is the one arch that runs the GPU at F32; see +[Known issues](#known-issues). Two effects explain the residuals that remain. The GPU plugin's F16 arithmetic is one. The other is SmolVLA on the NPU (1.7e-2), whose compile config turns on @@ -294,6 +313,8 @@ than the ggml contract, or fills a gap: | Fix | What it addresses | |---|---| | **PERMUTE op_case 2 requires a ROPE** | **assumes any permute of a view is a rope'd query** | +| **Two elementwise adds never stacked on a GEMM** | **the GPU plugin folds both in as post-ops and drops the second operand** | +| GPU inference precision exposed | the plugin's F16 default compounds through a denoise loop unrolled in one graph | | **GELU translated as tanh, not erf** | **assumes ggml's GELU is the exact erf form** | | Intel OpenCL platform selection | assumes the first OpenCL platform is Intel's | | RESHAPE `op_case` guard | assumes a reshape flattening dims 0-2 is the KV-cache flatten | @@ -301,18 +322,53 @@ than the ggml contract, or fills a gap: | **Position inputs keyed per tensor** | **assumes a graph has exactly one position input** | | Folded weights padded to full rank | a 2-D weight becomes a rank-2 constant, but views index it at ggml rank | | CONCAT input ranks aligned | same rank-2 constants, and concat cannot broadcast rank | -| Missing op translators | RELU, GELU_ERF, NEG, SQR had no table entry | +| Missing `GELU_ERF` translator | the exact-erf GELU op had no table entry at all, so a graph using it could not run | | Naive-path graph cache | that path re-compiled the whole model on every graph_compute, and its `graph_key` is a node count plus two names, which two graphs can share | | Interleaved-mrope sectors bounded | the sector cycle ignored `sections`, so the last few took the wrong stream | -| Interleaved-mrope mode passed through | the shared sin/cos table was built with the plain-rope layout | | Naive-path threshold settable | the 20-node constant is what picks the literal path | Five are worth expanding. **The PERMUTE op_case guard** is what makes GR00T N1.7 correct, and it is the subtlest of the set: a classifier that sent an ordinary head-split permute down -an LLM-specific rewrite. See [What is left](#what-is-left) for the bisect that -found it. +an LLM-specific rewrite. + +It used to return plausible-looking but wrong actions - max|delta| 1.477, 86% of +values off by more than 1e-2. Bisecting with cut-down graphs found it: truncating +the graph at a stage makes that stage the terminal node, which is the only way to +observe an interior tensor under this backend. Everything through the vision +tower, the LM and the vlsa stack was clean at 0.05-0.08%, and the error appeared +entirely inside the DiT action expert - specifically its cross-attention `V`, 139% +wrong while `K` from the same call was 0.04%. + +op_case 2 rewrites a tensor as `[n_seq, -1, n_heads, head_size]` before +transposing, which is right for llama.cpp's rope'd query and nothing else, but it +was reached by *any* permute whose source is a view of a non-leaf. GR00T N1.7's +`ggml_permute(view, 1,2,0,3)` over a fused KV projection took that path and came +out with its elements rearranged. `K` uses `permute(0,2,1,3)` and happened to +survive the same rewrite, which is why only `V` broke. Requiring an actual ROPE at +the end of the view/reshape/cont chain sends every other permute to op_case 1, the +plain transpose: 27% -> 0.005%, with every other arch bit-identical. + +**The double-elementwise guard** is what makes the iGPU usable for the Eagle-VLM +archs. The GPU plugin folds elementwise ops into the preceding GEMM as post-ops. +Given `ADD(ADD(residual, GEMM), graph_input)` it folds both, and the second +operand is silently lost - the result equals the inner add, as though the outer +one never ran. Nothing is logged. A llama.cpp graph never builds that chain, one +residual add per sub-block; a VLA does, wherever a tower's features are added on +top of an FFN residual. + +Found by bisecting VLA-JEPA with `GGML_OPENVINO_DEBUG_NODE`, which materialises +an arbitrary intermediate as an extra `ov::Result` - the only way to observe an +interior tensor here, since the backend writes back true graph outputs and +nothing else. Its ViT and DiT graphs matched the CPU plugin to 0.2%; the VLM +prefill was already wrong at the end of layer 0; and inside that layer a binary +search over the nodes landed on the FFN residual add sitting under the deepstack +add. Only the first three layers carry a deepstack add, which is why only those +three nodes mattered. Addition is associative, so the fix re-hangs the outer add +on the inner one's non-GEMM operand and the GEMM keeps a single post-op: +VLA-JEPA 5.4e-1 -> 2.6e-3 and GR00T N1.7 1.9e0 -> 2.6e-3, with the CPU plugin +unchanged. The same fusion path already had a known defect with broadcast `DIV`. **The GELU mode** is the highest-yield single fix in the list. ggml's `GGML_UNARY_OP_GELU` is the *tanh* approximation - its CPU kernel additionally @@ -360,6 +416,16 @@ build of the base commit, for every model tested. ## Known issues +**π0 needs F32 on the GPU, and gets it by default.** The GPU plugin computes in +F16, which is most of why it is fast. π0 unrolls its whole 10-step denoise loop +inside a single graph, so that error compounds across every step with nothing to +reset it: its continuous action dims land 4e-2 from an F32 reference, and its +gripper - a saturating ±1 channel - crosses its threshold one step late. On a +metric that reads as max|delta| 1.7; on a robot it is a late grasp. +`GGML_OPENVINO_GPU_PRECISION=f32` puts it back at 6.5e-5, and `backend_init` +defaults it for π0 alone because it costs about 3x (383 ms -> 1,170 ms). Set +`GGML_OPENVINO_GPU_PRECISION=f16` to override. No other arch needs it. + **Do not set `GGML_OPENVINO_CACHE_DIR`.** OpenVINO's on-disk blob cache reloads a compiled graph that computes the wrong thing. A cold run against a fresh cache directory is correct; the very next run, reading back the blobs it just wrote, is @@ -419,30 +485,11 @@ is therefore **untested**: only BitVLA emits it, and BitVLA never reaches this backend. It is in the table because it is a real gap in ggml-openvino, not because anything here exercises it. -**GR00T N1.7 was the last failure and is fixed.** It used to return -plausible-looking but wrong actions - max|delta| 1.477, 86% of values off by more -than 1e-2. Bisecting with cut-down graphs found it: truncating the graph at a -stage makes that stage the terminal node, which is the only way to observe an -interior tensor under this backend. Everything through the vision tower, the LM -and the vlsa stack was clean at 0.05-0.08%, and the error appeared entirely -inside the DiT action expert - specifically its cross-attention `V`, 139% wrong -while `K` from the same call was 0.04%. - -The cause was `compute_op_case`: PERMUTE op_case 2 rewrites a tensor as -`[n_seq, -1, n_heads, head_size]` before transposing, which is right for -llama.cpp's rope'd query and nothing else, but it was reached by *any* permute -whose source is a view of a non-leaf. GR00T N1.7's `ggml_permute(view, 1,2,0,3)` -over a fused KV projection took that path and came out with its elements -rearranged. `K` uses `permute(0,2,1,3)` and happened to survive the same rewrite, -which is why only `V` broke. Requiring an actual ROPE at the end of the -view/reshape/cont chain sends every other permute to op_case 1, the plain -transpose: 27% -> 0.005%, with every other arch bit-identical. - -**Untested archs.** π0 and OpenVLA-OFT are untested here for want of a local -checkpoint, not because anything is known to block them; both use op sets already -covered by tested archs (π0 matches π0.5, OpenVLA-OFT matches VLA-Adapter). Both -are `~` in the README matrix; treat any untested arch that way until it has -actually produced actions. +**Untested archs.** OpenVLA-OFT is untested here for want of a local checkpoint, +not because anything is known to block it; it uses an op set already covered by +VLA-Adapter. It is `~` in the README matrix; treat any untested arch that way until +it has actually produced actions. π0 has now been tested and translates faithfully +on the CPU plugin - it is one of the three that are wrong on the iGPU. **Splitting across devices.** Intel's own [π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py index c37aa53..0bbffba 100755 --- a/scripts/patch_ggml_openvino.py +++ b/scripts/patch_ggml_openvino.py @@ -23,6 +23,12 @@ those graphs take. Together they are what lets SmolVLA and pi0.5 run end to end on the CPU, GPU and NPU plugins. Numbers 4 and 5 are the ones worth upstreaming. +A hunk that applies is not a hunk that runs. Whenever VLA_LLAMA_TAG moves, check +that each patched function still has a live caller, not just that its anchor +still matches: b10729 moved the position-input naming out of the GgmlOvDecoder +member that fix 4 patches and into a free function, and the fix went silently +dead while applying cleanly. + See docs/backend/ov.md for the measured results and for what is still blocked. 1. ggml-openvino-extra.cpp - pick the *Intel* OpenCL platform. @@ -58,11 +64,16 @@ "Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32]) Argument shapes are inconsistent." When the graph has more than one, keep each tensor's own name. Nothing is - then called "inp_pos", so add_rope_sin_cos() returns early and the existing - fallback in translate_rope() builds sin/cos per op from its own position - input. Single-position graphs are untouched and keep the shared table. + then called "inp_pos", so translate_rope() falls back to building sin/cos + per op from its own position input. Single-position graphs are untouched. This one is what carries an arch through to a full prediction. + Patch the free function get_tensor_graph_input_ov_name() in + ggml-decoder.cpp, not just the GgmlOvDecoder member. b10729 relocated the + naming into that free function and left the member behind with no callers; + patching only the member applies cleanly and does nothing at all. Both are + patched here so the fix survives whichever one upstream keeps. + 5. utils.{h,cpp} - cache what the naive path compiles. The dynamic and static paths keep a `graph_key`-indexed cache of the decoder and the compiled infer request; the naive path has none, so it @@ -86,7 +97,9 @@ computed as erf, and an arch using the erf op could not run at all - and with no per-op CPU fallback in the core, "could not run" means the whole prediction. Small per node, but a vision tower has dozens and the error - compounds: fixing it is what moved GR00T N1.5 and VLA-JEPA inside the bar. + compounds: setting the mode explicitly moved VLA-JEPA from 5.5e-3 to 1.1e-4 + and GR00T N1.5 from 2.7e-2 to 6.0e-4, turning both from "runs but drifts" + into supported. The highest-yield single fix here. RELU, NEG and SQR were missing here too; upstream added all three in b10729. 7. openvino/utils.cpp - give a folded weight its full rank before slicing. @@ -117,15 +130,7 @@ because it removes a real divergence from the reference, not because a measurement demanded it. - 10. openvino/translate_session.cpp - pass the mode to the shared sin/cos table. - add_rope_sin_cos() called make_sin_cos() without the imrope flag, so a graph - with a single position input and interleaved mrope got a table built with - the plain-rope layout - silently wrong, not an error. Same caveat as 9: every - mrope arch here has several position inputs, so the shared precompute is - skipped and this path is untested. It is strictly closer to the reference - than what it replaces. - - 11. utils.cpp - make the naive-path graph-size threshold settable. + 10. utils.cpp - make the naive-path graph-size threshold settable. Graphs under 20 nodes bypass the LLM decoder and translate literally, with static shapes and no KV-cache inference. That literal path is the one that suits a vision tower, but a vision tower is ~450 nodes. The constant @@ -134,16 +139,7 @@ atoi turns junk into 0 and that would send every graph down the LLM builder with nothing said. - 12. openvino/op_table.cpp - translate GELU as the tanh approximation. - ggml's GGML_UNARY_OP_GELU is the *tanh* approximation (its CPU kernel also - reads an fp16 lookup table), but ov::op::v7::Gelu defaults to the exact erf - form and both ggml GELU ops were mapped onto that default. Per node the - difference is small; a Qwen3-VL vision tower has dozens of them and it - compounds. Setting the mode explicitly moved VLA-JEPA from 5.5e-3 to 1.1e-4 - and GR00T N1.5 from 2.7e-2 to 6.0e-4, turning both from "runs but drifts" - into supported. The highest-yield single fix here. - - 13. ggml-decoder.cpp - require a ROPE before taking PERMUTE op_case 2. + 11. ggml-decoder.cpp - require a ROPE before taking PERMUTE op_case 2. op_case 2 rewrites the tensor as [n_seq, -1, n_heads, head_size] and only then transposes, which is correct for llama.cpp's rope'd query and nothing else. The classifier reached it for ANY permute whose source is a view of a @@ -155,6 +151,37 @@ chain and requiring a ROPE at the end sends every other permute to op_case 1, the plain transpose. This is what makes GR00T N1.7 correct (27% -> 0.005%). + 12. ggml-decoder.cpp + openvino/op/add.cpp - do not stack two elementwise adds + on a GEMM. The GPU plugin folds elementwise ops into the preceding GEMM as + post-ops. Given ADD(ADD(residual, GEMM), graph_input) it folds both, and the + second operand is silently lost: the result equals the inner add, as though + the outer one never ran. Nothing is logged. An llama.cpp graph never builds + that chain - one residual add per sub-block - but a VLA does, wherever a + tower's features are added on top of an FFN residual. It cost VLA-JEPA, + GR00T N1.7 and pi0 their GPU support, while every other arch was unaffected. + Addition is associative, so re-hang the outer add on the inner one's + non-GEMM operand: the GEMM keeps exactly one post-op and the arithmetic is + unchanged. op_case 2/3 records which of the inner add's operands is the + GEMM, because ggml's operand order is not fixed. + Found by bisecting VLA-JEPA with GGML_OPENVINO_DEBUG_NODE: its ViT and DiT + graphs matched the CPU plugin to 0.2%, the VLM prefill was already wrong at + the end of layer 0, and within that layer the divergence was one node - the + FFN residual add under the deepstack add. The same fusion path already has a + known defect with broadcast DIV; see ggml_backend_openvino_supports_op. + + 13. ggml-openvino-extra.cpp - expose the GPU plugin's inference precision. + The GPU plugin computes in f16 unless told otherwise, which is most of why + it is fast, and for nearly every graph that is the right trade. It is not + the right trade for a graph carrying a long serial chain - a flow-matching + denoise loop unrolled inside one graph - because the error compounds across + every step with nothing to reset it, and a saturating output channel can + then cross its threshold in the wrong place. pi0 does exactly that: on the + GPU its continuous action dims land 4e-2 from an F32 reference and its + bistable gripper flips one step early, which is a 1.7 max|delta| on a metric + and a late grasp on a robot. GGML_OPENVINO_GPU_PRECISION=f32 puts it at + 6.5e-5. Exposed rather than forced: f32 costs about 3x on this plugin, so + src/backend.h defaults it for pi0 alone and an explicit setting still wins. + Idempotent - re-running on a patched tree is a no-op, so a reconfigure that re-populates the FetchContent source dir is safe either way. @@ -162,6 +189,7 @@ """ import pathlib +import re import sys HELPER = """// vla.cpp: select the Intel OpenCL platform. With several OpenCL runtimes @@ -353,6 +381,64 @@ ), (USM_LOOKUP % (("clEnqueueMemFillINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemFillINTEL",) * 2)), (USM_LOOKUP % (("clEnqueueMemcpyINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemcpyINTEL",) * 2)), + ( + """ "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + };""", + """ "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + // vla.cpp: f16 (default) or f32 for the GPU plugin's inference precision. + "GGML_OPENVINO_GPU_PRECISION", + };""", + ), + ( + """ } else if (cache_dir && strlen(cache_dir) > 0) { + compile_config.insert(ov::cache_dir(cache_dir)); + compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE)); + }""", + """ } else if (cache_dir && strlen(cache_dir) > 0) { + compile_config.insert(ov::cache_dir(cache_dir)); + compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE)); + } + + // vla.cpp: the GPU plugin computes in f16 unless told otherwise, which is why + // it is fast. A model whose graph carries a long serial chain -- a denoise + // loop unrolled inside one graph -- compounds that across every step, and a + // saturating output channel can then cross its threshold in the wrong place. + // Exposed rather than forced: f32 costs roughly 3x on this plugin. + if (device_name == "GPU") { + const char * gpu_prec = ggml_openvino_getenv_str("GGML_OPENVINO_GPU_PRECISION", "f16"); + if (strcmp(gpu_prec, "f32") == 0) { + compile_config.insert(ov::hint::inference_precision(ov::element::f32)); + } else if (strcmp(gpu_prec, "f16") != 0) { + GGML_LOG_WARN("GGML OpenVINO Backend: GGML_OPENVINO_GPU_PRECISION=%s is not f16 or f32, ignoring\\n", + gpu_prec); + } + }""", + ), + ], + "ggml/src/ggml-openvino/openvino/op/add.cpp": [ + ( + """ auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + auto res = std::make_shared(input_0, input_1);""", + """ auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + + // vla.cpp: re-hang the outer add on the inner one's non-GEMM operand so the + // GEMM is left with a single post-op. Addition is associative. + const int oc = context.get_op_case(); + if (oc == 2 || oc == 3) { + auto inner = input_0.get_node_shared_ptr(); + if (inner->get_input_size() == 2) { + const size_t keep = (oc == 2) ? 0 : 1; + const size_t fold = 1 - keep; + auto folded = std::make_shared(inner->input_value(fold), input_1); + auto res2 = std::make_shared(inner->input_value(keep), folded); + return rename_outputs_with_suffix({res2}, context.get_name()); + } + } + + auto res = std::make_shared(input_0, input_1);""", + ), ], "ggml/src/ggml-openvino/openvino/op/concat.cpp": [ ( @@ -525,6 +611,43 @@ ), ], "ggml/src/ggml-openvino/ggml-decoder.cpp": [ + ( + """ case GGML_OP_ADD: { + if (is_moe_expert_sum_add(node)) {""", + """ case GGML_OP_ADD: { + // vla.cpp: ADD(ADD(.., GEMM), graph-input). Two elementwise ops chained on a + // GEMM; the GPU plugin folds both in as post-ops and loses the second operand. + // op_case 2 = the inner add's input 0 is the GEMM, 3 = its input 1 is. + { + const ggml_tensor * inner = node->src[0]; + const ggml_tensor * other = node->src[1]; + if (inner && other && inner->op == GGML_OP_ADD && other->op == GGML_OP_NONE) { + for (int k = 0; k < 2; k++) { + const ggml_tensor * s = inner->src[k]; + while (s && s->src[0] && + (s->op == GGML_OP_VIEW || s->op == GGML_OP_RESHAPE || s->op == GGML_OP_CONT)) { + s = s->src[0]; + } + if (s && s->op == GGML_OP_MUL_MAT) { + op_case = (k == 0) ? 2 : 3; + } + } + } + } + if (is_moe_expert_sum_add(node)) {""", + ), + ( + """ if (GgmlOvDecoder::is_inp_pos(tensor, op)) { + return "inp_pos"; + }""", + """ if (GgmlOvDecoder::is_inp_pos(tensor, op)) { + // vla.cpp: this free function is the live naming path -- the + // GgmlOvDecoder member of the same intent is unreferenced at b10729. + // Only collapse ROPE position inputs onto one "inp_pos" parameter when + // the graph really has one. See scripts/patch_ggml_openvino.py. + return decoder->has_multiple_inp_pos() ? get_tensor_ov_name(cgraph, tensor) : std::string("inp_pos"); + }""", + ), ( """ } else { // rope'ed query tensor @@ -702,23 +825,19 @@ }();""", ), ], - "ggml/src/ggml-openvino/openvino/translate_session.cpp": [ - ( - '#include "translate_session.h"\n', - '#include "translate_session.h"\n\n#include "ggml.h" // vla.cpp: GGML_ROPE_TYPE_IMROPE\n', - ), - ( - " auto sin_cos = make_sin_cos(rope_params, inp_pos, rope_freqs_weight);", - """ // vla.cpp: rope_params[2] is the mode. The shared precompute never looked at - // it, so a graph with one position input and interleaved mrope got a table - // built with the plain-rope layout -- silently wrong actions, not an error. - const bool imrope = rope_params[2] == GGML_ROPE_TYPE_IMROPE; - auto sin_cos = make_sin_cos(rope_params, inp_pos, rope_freqs_weight, imrope, false);""", - ), - ], } +# A duplicate key in EDITS silently drops the earlier entry's hunks -- Python keeps +# the last one and nothing complains. That happened once and took the whole Intel +# OpenCL fix out of the patch, so compare the literal keys against the dict. +_keys = re.findall(r'^ "([^"]+)": \[$', pathlib.Path(__file__).read_text(), re.M) +if len(_keys) != len(EDITS): + _dup = sorted({k for k in _keys if _keys.count(k) > 1}) + sys.exit(f"patch_ggml_openvino: EDITS has {len(_keys)} literal keys but {len(EDITS)} entries; " + f"duplicate key(s): {_dup}") + + def main() -> int: root = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else ".") diff --git a/scripts/upstream_split.py b/scripts/upstream_split.py index 8a820e6..2b328f7 100755 --- a/scripts/upstream_split.py +++ b/scripts/upstream_split.py @@ -34,7 +34,22 @@ E = m.EDITS D = "ggml/src/ggml-openvino/" -# branch -> (subject, body, [(file, hunk-index), ...]) + +def H(rel, key): + """Address a hunk by a unique substring of its anchor, not by position. + + These used to be plain indices, and adding a hunk to the front of a file's + list silently handed every later hunk to the wrong branch -- the tail + coverage count still read 29/29, because each index was still used once. + A key that stops matching fails here instead of committing the wrong diff. + """ + hits = [i for i, (anchor, _) in enumerate(E[rel]) if key in anchor] + if len(hits) != 1: + sys.exit(f"upstream_split: key {key!r} matched {len(hits)} hunks in {rel}, expected 1") + return rel, hits[0] + + +# branch -> (subject, body, [H(file, anchor-substring), ...]) PRS = [ ("openvino-naive-cache", "openvino: cache the compiled model on the naive path", @@ -51,8 +66,10 @@ "compiled model is bound to the shapes it was built for, so a collision returns\n" "another graph's answer with no error. naive_key mixes in every node's op, type\n" "and shape. The map is bounded and flushed when full.", - [(D+"utils.h",0),(D+"utils.h",1),(D+"utils.h",2),(D+"utils.h",3), - (D+"utils.cpp",0),(D+"utils.cpp",1),(D+"utils.cpp",2)]), + [H(D+"utils.h","struct decoder_runtime_ctx"),H(D+"utils.h","graph_key_hash> decoder_cache"), + H(D+"utils.h","decoder_cache.clear()"),H(D+"utils.h","enum ggml_status naive_compute"), + H(D+"utils.cpp","if (!model_is_splitted)"),H(D+"utils.cpp","if (is_naive(cgraph))"), + H(D+"utils.cpp","enum ggml_status naive_compute")]), ("openvino-naive-graph-size-env", "openvino: make the naive-path graph-size threshold settable", @@ -63,7 +80,7 @@ "Expose the constant as GGML_OPENVINO_NAIVE_GRAPH_SIZE. Parsed with strtol and\n" "rejected loudly if it is not a whole positive number, because atoi turns junk\n" "into 0 and that would send every graph down the LLM builder with nothing said.", - [(D+"utils.cpp",3)]), + [H(D+"utils.cpp","bool is_naive(ggml_cgraph")]), ("openvino-gelu-modes", "openvino: map GELU to tanh and add GELU_ERF", @@ -74,7 +91,7 @@ "Small per node, but a vision tower has dozens and it compounds: on a ggml graph\n" "with a ViT encoder, fixing the mode moved two models from visibly wrong output\n" "to within 1.3e-3 of the CPU-backend reference.", - [(D+"openvino/op_table.cpp",0),(D+"openvino/op_table.cpp",1)]), + [H(D+"openvino/op_table.cpp","namespace ov {"),H(D+"openvino/op_table.cpp","{\"GGML_UNARY_OP_GELU\",")]), ("openvino-multiple-inp-pos", "openvino: stop distinct position inputs aliasing each other", @@ -87,10 +104,13 @@ " Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32])\n" " Argument shapes are inconsistent.\n\n" "When the graph has more than one, keep each tensor's own name. Nothing is then\n" - "called inp_pos, so add_rope_sin_cos() returns early and the existing fallback in\n" - "translate_rope() builds sin/cos per op from its own position input.\n" - "Single-position graphs are untouched and keep the shared table.", - [(D+"ggml-decoder.h",0),(D+"ggml-decoder.h",1),(D+"ggml-decoder.cpp",1)]), + "called inp_pos, so translate_rope() builds sin/cos per op from its own position\n" + "input. Single-position graphs are untouched.\n\n" + "Guard the free get_tensor_graph_input_ov_name() as well as the GgmlOvDecoder\n" + "member: the free function is the one compute_model_inputs() and\n" + "set_input_output() actually call, and the member currently has no callers.", + [H(D+"ggml-decoder.h","get_graph_input_ov_name"),H(D+"ggml-decoder.h","m_cgraph = nullptr"), + H(D+"ggml-decoder.cpp","is_inp_pos(tensor, op)"),H(D+"ggml-decoder.cpp","compute_op_case(const ggml_tensor")]), ("openvino-reshape-op-case", "openvino: narrow the RESHAPE op_case 3 guard", @@ -100,7 +120,37 @@ "inside ggml_conv_2d ([16,16,3,768] -> [768,768]) and rewrites it to the wrong\n" "shape. The real case always has node->ne[0] == 1; requiring that sends the conv\n" "kernel to case 6, the plain reshape.", - [(D+"ggml-decoder.cpp",0)]), + [H(D+"ggml-decoder.cpp","== node->ne[1]")]), + + ("openvino-gemm-double-eltwise", + "openvino: do not stack two elementwise adds on a GEMM", + "The GPU plugin folds elementwise ops into the preceding GEMM as post-ops. Given\n" + "ADD(ADD(residual, GEMM), graph_input) it folds both and the second operand is\n" + "silently lost: the result equals the inner add, as though the outer one never\n" + "ran. Nothing is logged.\n\n" + "A decoder-only LLM graph never builds that chain - one residual add per\n" + "sub-block - but a graph that adds a vision tower's features on top of an FFN\n" + "residual does, and three such models produced badly wrong output on GPU while\n" + "matching the CPU plugin to 1e-4.\n\n" + "Addition is associative, so re-hang the outer add on the inner one's non-GEMM\n" + "operand; the GEMM keeps exactly one post-op. op_case 2/3 records which operand\n" + "of the inner add is the GEMM, since the order is not fixed.\n\n" + "Same fusion path as the broadcast-DIV defect already handled in supports_op.", + [H(D+"ggml-decoder.cpp","case GGML_OP_ADD: {"), + H(D+"openvino/op/add.cpp","auto input_0 = process_view_input_new(context, 0);")]), + + ("openvino-permute-op-case", + "openvino: require a ROPE before taking PERMUTE op_case 2", + "op_case 2 rewrites the tensor as [n_seq, -1, n_heads, head_size] and only then\n" + "transposes, which is correct for llama.cpp's rope'd query and nothing else. The\n" + "classifier reaches it for ANY permute whose source is a view of a non-leaf.\n\n" + "A DiT cross-attention V built as ggml_permute(view, 1,2,0,3) over a fused KV\n" + "projection is therefore reshaped into a shape unrelated to it and comes out with\n" + "its elements rearranged, while a sibling K using permute(0,2,1,3) survives the\n" + "same rewrite -- 139% wrong against 0.04%, with no error reported.\n\n" + "Walk the view/reshape/cont chain and require a ROPE at the end; every other\n" + "permute falls to op_case 1, the plain transpose.", + [H(D+"ggml-decoder.cpp","rope'ed query tensor")]), ("openvino-sdpa-kv-f16", "openvino: convert K/V to F16 alongside Q in flash_attn_ext", @@ -108,7 +158,7 @@ "cache already is. A caller that keeps K/V in F32 hits OpenVINO's SDPA rejecting\n" "mixed input types (\"Mixed input types are not supported\"). Converting K/V too\n" "matches the precision the translator has already chosen for the other operands.", - [(D+"openvino/op/flash_attn_ext.cpp",0)]), + [H(D+"openvino/op/flash_attn_ext.cpp","q_f32, ov::element::f16")]), ("openvino-view-input-rank", "openvino: give a folded weight its full rank before slicing", @@ -119,7 +169,7 @@ " Axis 2 out of the tensor rank range [-2, 1].\n\n" "Reached by viewing Q, K and V out of one fused attn_in weight. Left-pad the\n" "input with leading 1s, which is the shape ggml gave it anyway.", - [(D+"openvino/utils.cpp",2),(D+"openvino/utils.cpp",3)]), + [H(D+"openvino/utils.cpp","openvino/op/transpose.hpp"),H(D+"openvino/utils.cpp","get_view_input_size(input_index)")]), ("openvino-concat-rank", "openvino: align CONCAT input ranks", @@ -128,7 +178,8 @@ "them. Hit by concatenating a CLS weight onto 4-D patch embeddings, and by\n" "concatenating a precomputed time tile onto a 4-D activation. Left-pad the\n" "shorter input with leading 1s before picking the axis.", - [(D+"openvino/op/concat.cpp",0),(D+"openvino/op/concat.cpp",1),(D+"openvino/op/concat.cpp",2)]), + [H(D+"openvino/op/concat.cpp","openvino/op/concat.hpp"),H(D+"openvino/op/concat.cpp","#include "), + H(D+"openvino/op/concat.cpp","rank - 1 - ggml_dim")]), ("openvino-imrope-sections", "openvino: bound the interleaved-mrope sector cycle by sections", @@ -140,15 +191,21 @@ "No measurable output change on the graphs tested, because the fourth stream\n" "happened to carry the same positions as the first. Submitted because it removes\n" "a divergence from the ggml reference, not because a measurement demanded it.", - [(D+"openvino/utils.cpp",0),(D+"openvino/utils.cpp",1)]), - - ("openvino-imrope-mode", - "openvino: pass the rope mode to the shared sin/cos table", - "add_rope_sin_cos() called make_sin_cos() without the imrope flag, so a graph with\n" - "a single position input and interleaved mrope got a table built with the\n" - "plain-rope layout. Silently wrong, not an error. The per-op path in\n" - "translate_rope() already passes the flag; this makes the shared precompute match.", - [(D+"openvino/translate_session.cpp",0),(D+"openvino/translate_session.cpp",1)]), + [H(D+"openvino/utils.cpp","#include "),H(D+"openvino/utils.cpp","gather_indices(n_dims_half)")]), + + ("openvino-gpu-precision-env", + "openvino: expose the GPU plugin's inference precision", + "The GPU plugin computes in f16 unless told otherwise, which is most of why it is\n" + "fast, and for nearly every graph that is the right trade.\n\n" + "It is not the right trade for a graph carrying a long serial chain, such as a\n" + "flow-matching denoise loop unrolled inside a single graph: the error compounds\n" + "across every step with nothing to reset it, and a saturating output channel can\n" + "then cross its threshold in the wrong place. One such model lands 4e-2 from an\n" + "F32 reference on GPU and 6.5e-5 with this set to f32.\n\n" + "Exposed rather than forced -- f32 costs roughly 3x -- and defaulted to f16, so\n" + "existing behaviour is unchanged unless the variable is set.", + [H(D+"ggml-openvino-extra.cpp","GGML_OPENVINO_LOG_UNSUPPORTED_OPS"), + H(D+"ggml-openvino-extra.cpp","} else if (cache_dir && strlen(cache_dir) > 0) {")]), ("openvino-intel-opencl-platform", "openvino: select the Intel OpenCL platform for the GPU remote context", @@ -159,8 +216,9 @@ "context. It aborts at startup with \"Incompatible OpenCL runtime: program is not\n" "in expected ELF format\".\n\n" "Select by CL_PLATFORM_VENDOR instead. Single-runtime hosts are unaffected.", - [(D+"ggml-openvino-extra.cpp",0),(D+"ggml-openvino-extra.cpp",1),(D+"ggml-openvino-extra.cpp",2), - (D+"ggml-openvino-extra.cpp",3),(D+"ggml-openvino-extra.cpp",4)]), + [H(D+"ggml-openvino-extra.cpp","#include "),H(D+"ggml-openvino-extra.cpp","ggml_openvino_device_config::init"), + H(D+"ggml-openvino-extra.cpp","cl_int err;"),H(D+"ggml-openvino-extra.cpp","clEnqueueMemFillINTEL_fn"), + H(D+"ggml-openvino-extra.cpp","clEnqueueMemcpyINTEL_fn")]), ] def git(*a): diff --git a/src/backend.h b/src/backend.h index 35b285c..9223fc5 100644 --- a/src/backend.h +++ b/src/backend.h @@ -47,6 +47,7 @@ #endif #include +#include #include #ifdef GGML_USE_OPENVINO #include @@ -240,6 +241,21 @@ inline Backend backend_init(const char * tag, int n_threads) { static std::once_flag naive_once; std::call_once(naive_once, [] { setenv_default("GGML_OPENVINO_NAIVE_GRAPH_SIZE", "1000000"); }); + // pi0 runs its whole 10-step denoise loop inside a single graph, so the + // GPU plugin's F16 arithmetic compounds across every step with nothing to + // reset it. On the continuous action dims that shows up as ~4e-2 against + // an F32 reference, and it is enough to move the bistable gripper channel + // across its threshold a step early -- which reads as a 1.7 error on a + // metric, and as the gripper closing late on a robot. Asking the GPU for + // F32 puts it back at 6.5e-5, and costs about 3x (383 ms -> 1170 ms). + // Only pi0 needs it: every other arch is inside the bar on the GPU at F16. + // A default, so GGML_OPENVINO_GPU_PRECISION=f16 still wins. + // Exact match: `tag` is the log prefix, and "vla(pi05)" contains "vla(pi0)", + // so anything looser would drag pi0.5 in too -- it does not need this. + if (tag && std::strcmp(tag, "vla(pi0)") == 0) { + setenv_default("GGML_OPENVINO_GPU_PRECISION", "f32"); + } + // ggml exposes OpenVINO as a single device, so VLA_DEVICE does not apply: // the target is chosen by name through GGML_OPENVINO_DEVICE (CPU / GPU / // NPU) and resolved inside ggml. From cb78ed68e7fd53229618d6de5874a9cf5f0286c3 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Tue, 1 Sep 2026 19:05:43 +0700 Subject: [PATCH 24/24] update docs and remove bug hunt scripts --- README.md | 3 - docs/backend/ov.md | 463 ++++++++++++++++----------------------------- scan.sh | 17 -- scan2.sh | 3 - scan_results.txt | 162 ---------------- scanner.py | 28 --- 6 files changed, 164 insertions(+), 512 deletions(-) delete mode 100755 scan.sh delete mode 100755 scan2.sh delete mode 100644 scan_results.txt delete mode 100644 scanner.py diff --git a/README.md b/README.md index 3574c9e..4a8a994 100644 --- a/README.md +++ b/README.md @@ -326,9 +326,6 @@ supported (released and benchmarked), `~` = in progress, `-` = planned. | [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | ~ | | [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | -Under OpenVINO, π0 runs the Intel iGPU at F32 rather than the default F16, which -`backend_init` sets for it - see [Known issues](docs/backend/ov.md#known-issues). - --- ## Contributing diff --git a/docs/backend/ov.md b/docs/backend/ov.md index 697ba90..a55d868 100644 --- a/docs/backend/ov.md +++ b/docs/backend/ov.md @@ -7,12 +7,10 @@ runtime on the configure line. > **Status: all nine tested architectures translate faithfully, on the CPU plugin > and on the iGPU.** SmolVLA, π0, π0.5, Evo-1, VLA-Adapter, GR00T N1.5, GR00T N1.6, -> GR00T N1.7 and VLA-JEPA all agree with a CPU-backend reference to 1.4e-3 or better -> on the OpenVINO CPU plugin - Evo-1 and VLA-Adapter to about 3e-6 - and eight of -> the nine are inside the same bar on the Arc B390 iGPU, where the speedup over the -> native CPU backend runs from 3.0x to 9.6x. VLA-Adapter is the one outside it -> there, at 5.3e-3 on actions peaking at 0.62, which is the plugin's F16 arithmetic -> rather than a translation error. +> GR00T N1.7 and VLA-JEPA all agree with a CPU-backend reference to 1.4e-3 or +> better on the OpenVINO CPU plugin, and eight of the nine are inside the same bar +> on the Arc B390 iGPU, where the speedup over the native CPU backend runs from +> 3.0x to 9.6x. > > Fifteen fixes were needed, thirteen of them inside ggml's OpenVINO backend, which > is written against llama.cpp's graphs and had never seen a vision tower or an @@ -24,39 +22,29 @@ runtime on the configure line. Measured on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10729`, -on the checkpoints under `vrfai/` on the Hub. The **fidelity** numbers for the CPU -and GPU plugins were all re-measured on that pin. The **latency** table and the -NPU column still date from `b10331` and are marked where they appear. +on the checkpoints under `vrfai/` on the Hub. Every **fidelity** number was +re-measured on that pin; only the **latency** table still dates from `b10331`. -OpenVINO is Intel's inference toolkit; ggml's backend translates a ggml compute -graph into an OpenVINO model and hands it to the CPU, GPU or NPU plugin, which -compiles and fuses it for the device. Unlike SYCL it needs no separate compiler: -the stock GCC/Clang build links `libopenvino` and everything else is ordinary -C++. +ggml's backend translates a ggml compute graph into an OpenVINO model and hands +it to the CPU, GPU or NPU plugin, which compiles and fuses it for the device. +Unlike SYCL it needs no separate compiler: the stock GCC/Clang build links +`libopenvino`. ## Supported devices -- Intel CPUs -- Intel GPUs (integrated Xe / Arc, and discrete) -- Intel NPUs (Core Ultra) +Intel CPUs, Intel GPUs (integrated Xe / Arc, and discrete), and Intel NPUs +(Core Ultra). Linux only here - Ubuntu 22.04 or 24.04. ## Prerequisites -Linux (Ubuntu 22.04 or 24.04) on Intel hardware. - ### 1. Device access The CPU plugin needs nothing. The GPU and NPU plugins reach the hardware through `/dev/dri/renderD*` and `/dev/accel/accel0`, both owned by the `render` group: ```bash -sudo usermod -aG render,video "$USER" -``` - -Re-login, then check that the OpenCL runtime actually enumerates the GPU: - -```bash -clinfo -l +sudo usermod -aG render,video "$USER" # re-login afterwards +clinfo -l # must enumerate the GPU ``` `Number of platforms 0` with `/etc/OpenCL/vendors/intel.icd` present almost @@ -66,86 +54,59 @@ always means the render group has not taken effect yet. Without it, For the GPU compute runtime and NPU driver packages themselves, follow [llama.cpp's OpenVINO notes](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md). -The NPU needs two more things that its driver packages do not pull in. Neither -failure is reported as an error - the device simply does not appear, and every -`GGML_OPENVINO_DEVICE=NPU` run lands on the CPU plugin instead: - -```text -GGML OpenVINO Backend: device NPU is not available, fallback to CPU -OpenVINO: using device CPU -``` - -**1. The Level Zero loader.** `intel-level-zero-npu` ships `libze_intel_npu.so.1`, -the *driver*; OpenVINO's NPU plugin only reaches it through the loader. - -```bash -sudo apt-get install -y libze1 # provides libze_loader.so.1 -``` - -`ldconfig -p | grep ze_loader` is the check. - -**2. Point the loader at the NPU driver.** Ubuntu's loader (1.16.1 in noble) -does not discover `libze_intel_npu.so.1` on its own, so installing it is not -enough by itself. Name the driver explicitly: +The NPU needs two more things its driver packages do not pull in. Neither failure +is reported as an error - the device simply does not appear and every +`GGML_OPENVINO_DEVICE=NPU` run lands on the CPU plugin instead +(`device NPU is not available, fallback to CPU`). ```bash +sudo apt-get install -y libze1 # the Level Zero loader; the driver alone is not enough + # check with: ldconfig -p | grep ze_loader export ZE_ENABLE_ALT_DRIVERS=/lib/x86_64-linux-gnu/libze_intel_npu.so.1 ``` -With that set the device enumerates as `NPU Intel(R) AI Boost` and the startup -banner reads `OpenVINO: using device NPU`. A loader from Intel's own graphics -repository, version-matched to the NPU driver, should discover it without the -override - untested here. +Ubuntu's loader (1.16.1 in noble) does not discover `libze_intel_npu.so.1` on its +own, hence the override; a loader from Intel's own graphics repository, +version-matched to the driver, should not need it - untested here. With both in +place the device enumerates as `NPU Intel(R) AI Boost`. ### 2. OpenVINO runtime + OpenCL headers ```bash -sudo apt-get install -y opencl-clhpp-headers ocl-icd-opencl-dev opencl-headers +sudo apt-get install -y opencl-clhpp-headers ocl-icd-opencl-dev opencl-headers \ + cmake ninja-build pkg-config protobuf-compiler libprotobuf-dev \ + libzmq3-dev cppzmq-dev ``` Then either install OpenVINO [from the archive](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-linux.html) -by hand, or run the bundled installer, which also pulls the GPU driver stack and -adds you to `render`: - -```bash -bash scripts/install_ov.sh -``` - -Plus the usual host dependencies: - -```bash -sudo apt-get install -y cmake ninja-build pkg-config \ - protobuf-compiler libprotobuf-dev libzmq3-dev cppzmq-dev -``` +by hand, or run `bash scripts/install_ov.sh`, which also pulls the GPU driver +stack and adds you to `render`. ## Configure & build ```bash source /opt/intel/openvino/setupvars.sh -cmake -B build-ov -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DGGML_OPENVINO=ON +cmake -B build-ov -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON cmake --build build-ov -j$(nproc) ``` -`setvars`-style sourcing is needed in every shell that builds *or* runs the -binaries: `libopenvino.so` and its TBB live under `/opt/intel`. Configure fails -early with a pointer back here if the runtime is not on `CMAKE_PREFIX_PATH`. +`setupvars.sh` must be sourced in every shell that builds *or* runs the binaries: +`libopenvino.so` and its TBB live under `/opt/intel`. Configure fails early with +a pointer back here if the runtime is not on `CMAKE_PREFIX_PATH`. `scripts/patch_ggml_openvino.py` runs as the FetchContent patch step, so the -thirteen ggml fixes described in its docstring are applied automatically. There is no -manual `git apply`. The step only runs when FetchContent populates the source -dir, so a `build/_deps` left over from an older checkout keeps the hunks it was -patched with: delete it after pulling rather than trusting a reconfigure. The -script checks each hunk on its own and fails loudly on a tree it cannot bring up -to date. +thirteen ggml fixes are applied automatically - there is no manual `git apply`. +The step only runs when FetchContent populates the source dir, so a `build/_deps` +left over from an older checkout keeps the hunks it was patched with: delete it +after pulling rather than trusting a reconfigure. The script checks each hunk on +its own and fails loudly on a tree it cannot bring up to date. ## Run -`GGML_OPENVINO_DEVICE` picks the target by name. Do not type the placeholder -`` literally - in a shell the angle brackets are input redirection. +`GGML_OPENVINO_DEVICE` picks the target by name (`VLA_DEVICE` does *not* apply - +ggml exposes OpenVINO as a single device): ```bash GGML_OPENVINO_DEVICE=GPU ./build-ov/vla-server ./weights/smolvla-libero.gguf @@ -160,32 +121,25 @@ vla: backend = OPENVINO (asked for GPU, see ggml's "using device" line) The first comes from ggml and is authoritative: an unavailable device logs a warning there and falls back to `CPU`, which is still the OpenVINO CPU plugin, -not ggml's native CPU backend. The second line echoes what was requested, so the -pair tells you whether you got the device you asked for. `VLA_DEVICE` does *not* -apply - ggml exposes OpenVINO as a single device and the target is chosen by -name. +not ggml's native CPU backend. The second echoes what was requested, so the pair +tells you whether you got the device you asked for. OpenVINO compiles each graph on first use, which is slow - a minute or two for a vision tower on the GPU. Compiled graphs are then cached in-process for the life of the model, so only the first prediction pays that; give any client a receive -timeout well above the first request. - -Do **not** set `GGML_OPENVINO_CACHE_DIR` to carry them across restarts. It -produces silently wrong actions here - see -[Known issues](#known-issues). `backend_init` clears it and says so; -`VLA_ALLOW_OV_CACHE=1` keeps it if you have verified the outputs yourself. +timeout well above the first request. Do **not** set `GGML_OPENVINO_CACHE_DIR` to +carry them across restarts; it produces silently wrong actions here - see +[Known issues](#known-issues). ## Results `vla_predict_check` (a test target - add `-DVLA_BUILD_TESTS=ON`), fixed noise, one camera view, best of 4-6 iterations after 3 warmups. "CPU backend" is ggml's -own CPU backend on the same 16-core host. No `GGML_OPENVINO_CACHE_DIR`, for the -reason in [Known issues](#known-issues). +own CPU backend on the same 16-core host. No `GGML_OPENVINO_CACHE_DIR`. -These latencies were taken at llama.cpp `b10331` and have not been re-timed on -`b10729`; the fidelity numbers under [Full results](#full-results) have. Read the -GPU column for VLA-JEPA and GR00T N1.7 as the cost of a wrong answer - see -[Known issues](#known-issues). +Latencies were taken at `b10331` and have not been re-timed on `b10729`. Read the +GPU column for VLA-JEPA and GR00T N1.7 as the cost of a wrong answer at that pin; +both are correct now. | Model | input | CPU backend | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | |---|---|---:|---:|---:|---:| @@ -209,16 +163,13 @@ NPU limits under [Known issues](#known-issues). Actions are checked against the CPU backend on identical inputs; both sides are deterministic, so the numbers are exact rather than sampled. But **which** CPU run -you compare against matters more than it looks. - -OpenVINO folds the checkpoint's BF16 weights in as constants and its CPU plugin -executes them at F32. ggml's CPU backend, on the same checkpoint, keeps them BF16. -So a naive comparison charges the OpenVINO backend for a precision *upgrade*. -Running the reference with `--weight-dtype f32` removes that term. +you compare against matters. OpenVINO folds the checkpoint's BF16 weights in as +constants and its CPU plugin executes them at F32, while ggml's CPU backend keeps +them BF16, so a naive comparison charges the OpenVINO backend for a precision +*upgrade*. Running the reference with `--weight-dtype f32` removes that term. -The two references bracket the answer, and which one is tighter is arch-dependent -- it turns on how much of a given checkpoint is BF16 in the first place. Report -both and take the smaller as the fidelity figure: +The two references bracket the answer, and which is tighter turns on how much of +a given checkpoint is BF16 in the first place. Report both and take the smaller: | Model | vs BF16 reference | vs F32 reference | tighter reference | |---|---:|---:|---| @@ -232,12 +183,10 @@ both and take the smaller as the fidelity figure: | GR00T N1.7 | 1.5e0 | **4.2e-4** | F32 | Evo-1 and VLA-Adapter agree with an F32 reference to six decimal places, which is -as close to "the translation is exact" as this harness can show - for those two, -OpenVINO is doing F32 arithmetic and the BF16 comparison was measuring nothing but -the dtype. SmolVLA is the counterexample that stops this being a universal rule: it lands -closer to the BF16 reference, so its checkpoint evidently is not uniformly BF16 -where it matters. For context, the -CPU backend's own output moves by 2.0e-3 (SmolVLA), 2.7e-3 (Evo-1) or 1.1e-2 +as close to "the translation is exact" as this harness can show; the BF16 +comparison for those two was measuring nothing but the dtype. SmolVLA is the +counterexample that stops this being a universal rule. For context, the CPU +backend's own output moves by 2.0e-3 (SmolVLA), 2.7e-3 (Evo-1) or 1.1e-2 (VLA-JEPA) when you flip that one flag, so the model's intrinsic sensitivity to precision is the same size as the numbers being reported. @@ -245,58 +194,51 @@ precision is the same size as the numbers being reported. Against the F32 reference, which is the fidelity number: -| Model | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU (`b10331`) | +| Model | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | |---|---:|---:|---:| -| Evo-1 | 2.2e-6 | 6.0e-4 | not supported | -| VLA-Adapter | 3.9e-6 | 6.8e-3 | not supported | -| π0.5 | 6.1e-5 | 7.6e-4 | 9.9e-4 | +| Evo-1 | 2.2e-6 | 6.0e-4 | compiler rejects | +| VLA-Adapter | 3.9e-6 | 6.8e-3 | compiler rejects | +| π0.5 | 6.1e-5 | 7.6e-4 | 1.6e-3 | | VLA-JEPA | 7.7e-5 | 2.6e-3 | returns NaN | -| GR00T N1.7 | 3.9e-4 | 2.6e-3 | not attempted | -| π0 | 5.8e-4 | 6.6e-5 | not attempted | -| GR00T N1.5 | 6.0e-4 | 4.6e-3 | plugin throws | -| GR00T N1.6 | 1.1e-3 | 1.4e-3 | plugin throws | -| SmolVLA | 1.4e-3 | 2.6e-3 | 1.7e-2 | +| GR00T N1.7 | 3.9e-4 | 2.6e-3 | NPUW throws | +| π0 | 5.8e-4 | 6.6e-5 | **1.7e0** | +| GR00T N1.5 | 6.0e-4 | 4.6e-3 | NPUW throws | +| GR00T N1.6 | 1.1e-3 | 1.4e-3 | NPUW throws | +| SmolVLA | 1.4e-3 | 2.6e-3 | 1.1e-2 | Every tested arch is inside the 2.9e-3 bar on the CPU plugin, most by one to three orders of magnitude, and eight of the nine are inside it on the GPU as well (GR00T N1.5's 4.6e-3 against F32 is 1.6e-3 against BF16, the tighter reference for -that arch). - -The one that sits outside is **VLA-Adapter**, at 6.8e-3 against F32 and 5.3e-3 -against BF16 on actions peaking at 0.62. That is the GPU plugin computing in F16 -and is a precision effect, not a translation error: the same arch is 3.9e-6 on the -CPU plugin. Judge translation fidelity on the CPU plugin and treat the GPU as a -separate precision target. - -π0 is the exception in the other direction - it is *tighter* on the GPU (6.6e-5) -than on the CPU plugin, because it is the one arch that runs the GPU at F32; see -[Known issues](#known-issues). - -Two effects explain the residuals that remain. The GPU plugin's F16 arithmetic is -one. The other is SmolVLA on the NPU (1.7e-2), whose compile config turns on -dynamic quantization - π0.5 on the same device stays at 9.9e-4, so that is a -property of the model on that device rather than of the backend. +that arch). The one outside is **VLA-Adapter**, at 6.8e-3 against F32 on actions +peaking at 0.62 - the GPU plugin computing in F16, a precision effect rather than +a translation error, since the same arch is 3.9e-6 on the CPU plugin. Judge +translation fidelity on the CPU plugin and treat the GPU as a separate precision +target. π0 is the exception in the other direction, *tighter* on the GPU (6.6e-5) +because it is the one arch that runs the GPU at F32; see +[Known issues](#known-issues), which also covers the NPU column. ## What had to change -Two fixes on the vla.cpp side. Both are ordinary correctness fixes that happen to -be invisible on the other backends: +Fifteen fixes: two in vla.cpp, thirteen in ggml's OpenVINO backend. Both +vla.cpp-side ones are ordinary correctness fixes that happen to be invisible on +the other backends - `vla_predict_check` on a CPU build of this branch is +byte-identical to the same build of the base commit, for every model tested. - **Weight buffers are tagged.** `ggml_backend_alloc_ctx_tensors` leaves a buffer on `GGML_BACKEND_BUFFER_USAGE_ANY`, and ggml-openvino reads ANY as "KV cache", giving every weight a dynamic sequence dimension. `vla::alloc_weights` in [`src/backend.h`](../../src/backend.h) tags it `..._WEIGHTS`, which is what - llama.cpp does with its own weights and what lets the frontend fold them in as - constants. Since 0.3.0 every arch allocates through `vla::WeightLoader`, so - this is one call site in [`src/loader.cpp`](../../src/loader.cpp). + lets the frontend fold weights in as constants. Since 0.3.0 every arch + allocates through `vla::WeightLoader`, so this is one call site in + [`src/loader.cpp`](../../src/loader.cpp). - **Graph tensors get unique names.** ggml derives a result's name from its - source, so `ggml_reshape_2d` of an unnamed tensor is called `" (reshaped)"` - - and a graph whose intermediates were never named ends up with many tensors - sharing one name. ggml-openvino keys its translation map on those names, so - duplicates silently collapse into one node and the graph wires the wrong tensor - into the next op. `vla::graph_unique_names` relabels duplicates before compute, - at each of the 29 `ggml_backend_graph_compute` call sites. It compiles to - nothing outside an OpenVINO build. + source, so a graph whose intermediates were never named ends up with many + tensors sharing one name (`" (reshaped)"` and friends). ggml-openvino keys its + translation map on those names, so duplicates silently collapse into one node + and the graph wires the wrong tensor into the next op. `vla::graph_unique_names` + relabels duplicates before compute, at each of the 29 + `ggml_backend_graph_compute` call sites. It compiles to nothing outside an + OpenVINO build. `backend_init` also sets one default, the way the SYCL rung already sets `GGML_SYCL_ENABLE_VMM=0`: **`GGML_OPENVINO_NAIVE_GRAPH_SIZE` defaults high.** @@ -305,10 +247,10 @@ larger through a model builder that assumes a decoder-only LLM. The literal path is the one that fits a vision tower and an action expert. An explicit setting still wins. -The other thirteen are in ggml's OpenVINO backend itself, applied by -`scripts/patch_ggml_openvino.py` at configure time. Its docstring carries the -detail; in short each narrows an llama.cpp-shaped assumption that is stricter -than the ggml contract, or fills a gap: +The other thirteen are applied to ggml's OpenVINO backend by +`scripts/patch_ggml_openvino.py` at configure time; its docstring carries the +per-fix detail. Each narrows an llama.cpp-shaped assumption that is stricter than +the ggml contract, or fills a gap: | Fix | What it addresses | |---|---| @@ -327,181 +269,104 @@ than the ggml contract, or fills a gap: | Interleaved-mrope sectors bounded | the sector cycle ignored `sections`, so the last few took the wrong stream | | Naive-path threshold settable | the 20-node constant is what picks the literal path | -Five are worth expanding. - -**The PERMUTE op_case guard** is what makes GR00T N1.7 correct, and it is the -subtlest of the set: a classifier that sent an ordinary head-split permute down -an LLM-specific rewrite. - -It used to return plausible-looking but wrong actions - max|delta| 1.477, 86% of -values off by more than 1e-2. Bisecting with cut-down graphs found it: truncating -the graph at a stage makes that stage the terminal node, which is the only way to -observe an interior tensor under this backend. Everything through the vision -tower, the LM and the vlsa stack was clean at 0.05-0.08%, and the error appeared -entirely inside the DiT action expert - specifically its cross-attention `V`, 139% -wrong while `K` from the same call was 0.04%. - -op_case 2 rewrites a tensor as `[n_seq, -1, n_heads, head_size]` before -transposing, which is right for llama.cpp's rope'd query and nothing else, but it -was reached by *any* permute whose source is a view of a non-leaf. GR00T N1.7's -`ggml_permute(view, 1,2,0,3)` over a fused KV projection took that path and came -out with its elements rearranged. `K` uses `permute(0,2,1,3)` and happened to -survive the same rewrite, which is why only `V` broke. Requiring an actual ROPE at -the end of the view/reshape/cont chain sends every other permute to op_case 1, the -plain transpose: 27% -> 0.005%, with every other arch bit-identical. - -**The double-elementwise guard** is what makes the iGPU usable for the Eagle-VLM -archs. The GPU plugin folds elementwise ops into the preceding GEMM as post-ops. -Given `ADD(ADD(residual, GEMM), graph_input)` it folds both, and the second -operand is silently lost - the result equals the inner add, as though the outer -one never ran. Nothing is logged. A llama.cpp graph never builds that chain, one -residual add per sub-block; a VLA does, wherever a tower's features are added on -top of an FFN residual. - -Found by bisecting VLA-JEPA with `GGML_OPENVINO_DEBUG_NODE`, which materialises -an arbitrary intermediate as an extra `ov::Result` - the only way to observe an -interior tensor here, since the backend writes back true graph outputs and -nothing else. Its ViT and DiT graphs matched the CPU plugin to 0.2%; the VLM -prefill was already wrong at the end of layer 0; and inside that layer a binary -search over the nodes landed on the FFN residual add sitting under the deepstack -add. Only the first three layers carry a deepstack add, which is why only those -three nodes mattered. Addition is associative, so the fix re-hangs the outer add -on the inner one's non-GEMM operand and the GEMM keeps a single post-op: -VLA-JEPA 5.4e-1 -> 2.6e-3 and GR00T N1.7 1.9e0 -> 2.6e-3, with the CPU plugin -unchanged. The same fusion path already had a known defect with broadcast `DIV`. - -**The GELU mode** is the highest-yield single fix in the list. ggml's -`GGML_UNARY_OP_GELU` is the *tanh* approximation - its CPU kernel additionally -reads an fp16 lookup table - while `ov::op::v7::Gelu` defaults to the exact erf -formulation, and both ggml GELU ops were mapped onto that default. The error per -node is small, but a Qwen3-VL vision tower contains dozens of them and it -compounds through the encoder. Setting the mode explicitly moved VLA-JEPA from -5.5e-3 to 1.1e-4 (48x) and GR00T N1.5 from 2.7e-2 to 6.0e-4 (45x), turning both -from "runs but drifts" into supported, and improved GR00T N1.6 and π0.5 too. It -is worth upstreaming alongside the position-input fix. - -**Position inputs** is what carries an arch through to a full prediction. Every -tensor feeding a `GGML_OP_ROPE`'s second input was renamed to a single parameter -called `inp_pos`, and one shared sin/cos table was built from it. SmolVLA passes -three position tensors - prefill, full and rebased - so they aliased each other -and every RoPE took the table built from whichever won: - -```text -opset1::Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32]) -Argument shapes are inconsistent. -``` +The bolded rows are the ones that turned a wrong arch into a correct one: +PERMUTE op_case 2 for GR00T N1.7, the double-elementwise guard for the Eagle-VLM +archs on the iGPU, the GELU mode for VLA-JEPA and GR00T N1.5, per-tensor position +inputs for SmolVLA (which passes three position tensors, so they aliased). The +GELU mode and the position-input fix are the two worth upstreaming. -When the graph has more than one, each keeps its own name. Nothing is then called -`inp_pos`, the shared-table precompute returns early, and `translate_rope()` -falls back to building sin/cos per op from its own position input - a path that -already existed for mixed RoPE parameters. Single-position graphs are untouched. - -**Rank padding** is the other structural one. A ggml tensor that is 2-D folds in -as a rank-2 constant, which is what a GEMM operand wants, but the graph indexes -it at full ggml rank. Evo-1 views Q, K and V out of one fused `attn_in` weight -and got `Axis 2 out of the tensor rank range [-2, 1]`. Padding in -`process_view_input_new` fixed that class generally, and padding in the concat -translator is what saves each arch from working around it - SmolVLA would -otherwise need its time tiles moved into a buffer of their own. - -**The naive-path cache** is about speed, not correctness. The dynamic and static -paths keep a `graph_key`-indexed cache; the naive path had none, so it rebuilt -the decoder, re-converted the model and called `compile_model()` on every single -`ggml_backend_graph_compute`. SmolVLA on the CPU plugin ran at 22.7 s per -prediction before, 1.4 s after. - -None of the vla.cpp-side changes alter what the other backends compute: -`vla_predict_check` on a CPU build of this branch is byte-identical to the same -build of the base commit, for every model tested. +**Debugging a mistranslation.** The backend writes back true graph outputs and +nothing else, so an interior tensor is observable only two ways: truncate the +graph at a stage, which makes that stage the terminal node, or set +`GGML_OPENVINO_DEBUG_NODE` to materialise one node as an extra `ov::Result`. Both +of the subtlest fixes above were found by bisecting that way, comparing each +stage against a CPU-backend reference; neither logs anything when it goes wrong. ## Known issues **π0 needs F32 on the GPU, and gets it by default.** The GPU plugin computes in F16, which is most of why it is fast. π0 unrolls its whole 10-step denoise loop -inside a single graph, so that error compounds across every step with nothing to -reset it: its continuous action dims land 4e-2 from an F32 reference, and its -gripper - a saturating ±1 channel - crosses its threshold one step late. On a -metric that reads as max|delta| 1.7; on a robot it is a late grasp. -`GGML_OPENVINO_GPU_PRECISION=f32` puts it back at 6.5e-5, and `backend_init` -defaults it for π0 alone because it costs about 3x (383 ms -> 1,170 ms). Set -`GGML_OPENVINO_GPU_PRECISION=f16` to override. No other arch needs it. +inside a single graph, so that error compounds with nothing to reset it: its +continuous action dims land 4e-2 from an F32 reference and its gripper - a +saturating ±1 channel - crosses its threshold one step late. That reads as +max|delta| 1.7; on a robot it is a late grasp. `GGML_OPENVINO_GPU_PRECISION=f32` +puts it back at 6.5e-5, and `backend_init` defaults it for π0 alone because it +costs about 3x (383 ms -> 1,170 ms). Set `=f16` to override. No other arch needs +it. **Do not set `GGML_OPENVINO_CACHE_DIR`.** OpenVINO's on-disk blob cache reloads a compiled graph that computes the wrong thing. A cold run against a fresh cache directory is correct; the very next run, reading back the blobs it just wrote, is -not: - -```text -GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_CACHE_DIR=$dir # cold: max |delta| 1.2e-3 -GGML_OPENVINO_DEVICE=GPU GGML_OPENVINO_CACHE_DIR=$dir # warm: max |delta| 2.9e0 -``` - -Nothing is logged - the actions are simply wrong, which for a policy server is -the worst possible failure mode. `backend_init` therefore clears the variable and -says so; set `VLA_ALLOW_OV_CACHE=1` alongside it to keep it. Unverified guess at -the cause: the blob key does not capture something that differs between -vla.cpp's several graphs, so one graph gets another's blob - the same class of -bug as the in-process `graph_key` above, which is now keyed on shapes. In practice, pay the compile once per process and leave it unset. - -**The NPU takes two of the eight archs, and fails four different ways.** SmolVLA -and π0.5 run. The others do not: +not - max|delta| 1.2e-3 cold, 2.9e0 warm, with nothing logged, which for a policy +server is the worst possible failure mode. `backend_init` therefore clears the +variable and says so; `VLA_ALLOW_OV_CACHE=1` keeps it. Unverified guess at the +cause: the blob key does not capture something that differs between vla.cpp's +several graphs, so one graph gets another's blob - the same class of bug as the +in-process `graph_key` above, which is now keyed on shapes. In practice, pay the +compile once per process and leave it unset. + +**The NPU accepts three of the nine archs, and only two of those are correct.** +Check every NPU run against the startup banner: an unavailable NPU falls back to +the CPU plugin silently and would otherwise report excellent numbers that are not +NPU numbers at all. A partially-failing run still reports a wall-clock time, so +do not read a latency off a run whose actions did not come out. | Model | NPU outcome | |---|---| +| SmolVLA | runs, 1.1e-2 (the compile config's dynamic quantization) | +| π0.5 | runs, 1.6e-3 | +| π0 | runs, but **1.7e0 wrong** - see below | +| VLA-JEPA | compiles and runs, returns all `NaN` (not diagnosed) | +| GR00T N1.5 / N1.6 / N1.7 | `NPUW: Assertion all_ok failed`, `partitioning.cpp:1350` | | Evo-1 | compiler rejects: `Input channels '1025' is not aligned by '16'` | | VLA-Adapter | compiler rejects: `Input channels '261' is not aligned by '16'` | -| VLA-JEPA | compiles and runs, returns all `NaN` | -| GR00T N1.5 | NPUW partitioning throws (`partitioning.cpp:1350`) | -| GR00T N1.6 | plugin throws (`core.cpp:117`) | -| GR00T N1.7 | not attempted | - -Five archs, four distinct failures, none of them vla.cpp's. The two alignment rejections are -Intel's NPU compiler: 1025 is Evo-1's 1024 -patches plus a CLS token, 261 is VLA-Adapter's 256 plus 5, and neither is a -multiple of 16. SmolVLA and π0.5 happen to have 16-aligned sequence lengths. The -VLA-JEPA NaN is a third failure mode and is not diagnosed. Note that a -partially-failing NPU run still reports a wall-clock time, so do not read a -latency number off a run whose actions did not come out. + +The two alignment rejections are Intel's NPU compiler: 1025 is Evo-1's 1024 +patches plus a CLS token, 261 is VLA-Adapter's 256 plus 5. SmolVLA and π0.5 +happen to have 16-aligned sequence lengths. None of these are vla.cpp's doing. + +**π0 on the NPU is the same bug as π0 on the GPU, and here there is no remedy.** +Continuous dims 0-5 land at 4.0e-2 and the gripper flips at step 44, exactly as +on the GPU. But the GPU fix does not transfer: setting the inference precision to +F32 makes the NPU refuse to compile at all (`core.cpp:117`), because the hint +conflicts with the NPUW and dynamic-quantization config the NPU path sets up. So +`GGML_OPENVINO_GPU_PRECISION` is GPU-only by necessity, and π0 should not be run +on the NPU. **SmolVLA's `VLA_TIMING=phase` path is wrong under OpenVINO.** SmolVLA has a second graph builder used when a caller asks for per-phase timings, and it does -not survive translation - max |delta| 1.9 on every device, with or without the -in-process cache. The default `TimingDetail::NONE` path, which is what -`vla-server` and `vla-cli` use, is correct, and on the native CPU backend the two -paths agree exactly. Evo-1 and π0.5 are unaffected on the same path, so this is -specific to SmolVLA's second graph. One hypothesis - that the split-graph guard -sends it down the LLM path - was tested and is wrong: forcing the naive path on -split graphs returns zeros. Per-stage timings for SmolVLA are therefore omitted -from the tables above. +not survive translation - max|delta| 1.9 on every device. The default +`TimingDetail::NONE` path, which is what `vla-server` and `vla-cli` use, is +correct, and on the native CPU backend the two paths agree exactly. Evo-1 and +π0.5 are unaffected on the same path, so this is specific to SmolVLA's second +graph; one hypothesis - that the split-graph guard sends it down the LLM path - +was tested and is wrong. Per-stage timings for SmolVLA are omitted from the +tables above. ## What is left **Op coverage is no longer the blocker.** With RELU, GELU_ERF, NEG and SQR added to the table, every ggml op the eleven in-tree archs build is translatable. The -one exception is `ggml_map_custom1`, used only by BitVLA - and BitVLA pins its -ggml graph to the CPU backend by design and offloads its LM through hand-written -CUDA kernels, so an OpenVINO build leaves it on the CPU regardless. `GGML_OP_SQR` -is therefore **untested**: only BitVLA emits it, and BitVLA never reaches this -backend. It is in the table because it is a real gap in ggml-openvino, not -because anything here exercises it. - -**Untested archs.** OpenVLA-OFT is untested here for want of a local checkpoint, -not because anything is known to block it; it uses an op set already covered by -VLA-Adapter. It is `~` in the README matrix; treat any untested arch that way until -it has actually produced actions. π0 has now been tested and translates faithfully -on the CPU plugin - it is one of the three that are wrong on the iGPU. +one exception is `ggml_map_custom1`, used only by BitVLA, which pins its ggml +graph to the CPU backend by design - so an OpenVINO build leaves it there +regardless. That also makes `GGML_OP_SQR` **untested**: only BitVLA emits it. It +is in the table because it is a real gap in ggml-openvino, not because anything +here exercises it. + +**Untested archs.** OpenVLA-OFT is untested for want of a local checkpoint, not +because anything is known to block it; it uses an op set already covered by +VLA-Adapter. It is `~` in the README matrix; treat any untested arch that way +until it has actually produced actions. **Splitting across devices.** Intel's own [π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) puts the vision encoder and language model on the iGPU and the action expert on -the NPU, with the KV cache as the only cross-device handoff. That is a different -toolchain - PyTorch exported to OpenVINO IR as three separate models, no ggml - -so none of it drops into this backend. What carries over is the shape of the -answer: the two devices suit different stages, and π0.5 on the NPU alone is -already within 1.5x of the iGPU at a fraction of the power. - -vla.cpp cannot make that split today because the core drives one backend for a -whole prediction. It would need a per-*stage* backend rather than a per-op +the NPU, with the KV cache as the only cross-device handoff. The toolchain does +not carry over - PyTorch exported to OpenVINO IR as three separate models, no +ggml - but the shape of the answer does: the two devices suit different stages, +and π0.5 on the NPU alone is already within 1.5x of the iGPU at a fraction of the +power. vla.cpp cannot make that split today because the core drives one backend +for a whole prediction. It would need a per-*stage* backend rather than a per-op scheduler - the vision tower, the prefix and the action expert already hand off through host memory, so the seam is in the right place - but that is an engine change, not a backend one. diff --git a/scan.sh b/scan.sh deleted file mode 100755 index cdca098..0000000 --- a/scan.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -echo "=== INT OVERFLOWS ===" -rg "(int|int32_t)\s+[a-zA-Z0-9_]+\s*=\s*[a-zA-Z0-9_>.\-]*ne\[[0-3]\]\s*\*" src/ -rg "(int|int32_t)\s+[a-zA-Z0-9_]+\s*=\s*.*\*.*ne\[[0-3]\]" src/ -echo "=== FOPEN WITHOUT FCLOSE ===" -rg -l "fopen" src/ | xargs -I{} bash -c 'grep -q fclose {} || echo {} missing fclose' -echo "=== GGML_NEW_CONTEXT WITHOUT GGML_FREE ===" -rg -l "ggml_init" src/ | xargs -I{} bash -c 'grep -q ggml_free {} || echo {} missing ggml_free' -echo "=== BACKEND BUFFER MEMORY ===" -rg -l "ggml_backend_alloc_buffer" src/ | xargs -I{} bash -c 'grep -q ggml_backend_buffer_free {} || echo {} missing buffer free' -echo "=== SINGLE THREADED LOOPS ===" -rg -i "for.*y.*height.*for.*x.*width" src/ -rg -i "for.*i.*<.*w.*h" src/ -echo "=== F32 ROUND TRIPS ===" -rg "ggml_cast.*F32" src/ -rg "ggml_cpy.*F32" src/ -echo "=== UNCHECKED TENSOR SHAPES ===" diff --git a/scan2.sh b/scan2.sh deleted file mode 100755 index e1b5d30..0000000 --- a/scan2.sh +++ /dev/null @@ -1,3 +0,0 @@ -rg -n "\bmalloc\(" src/ -rg -n "\bnew\b" src/ -rg -n "\bfopen\(" src/ diff --git a/scan_results.txt b/scan_results.txt deleted file mode 100644 index 213f55d..0000000 --- a/scan_results.txt +++ /dev/null @@ -1,162 +0,0 @@ - ---- Checking src/model.cpp --- -BUGS_FILE src/model.cpp:46 | gguf_context * gctx = gguf_init_from_file(path.c_str(), p); - ---- Checking src/loader.cpp --- - ---- Checking src/vla_c_api.cpp --- - ---- Checking src/options.cpp --- - ---- Checking src/vlm/engine.cpp --- - ---- Checking src/serving/server.cpp --- - ---- Checking src/serving/vla-cli.cpp --- -BUGS_FILE src/serving/vla-cli.cpp:172 | FILE * fp = popen(cmd.c_str(), "r"); - ---- Checking src/serving/vlm-server.cpp --- - ---- Checking src/serving/vla-bench.cpp --- - ---- Checking src/models/bitvla.cpp --- -BUGS_FILE src/models/bitvla.cpp:596 | if (!g.open(ckpt_path)) -BUGS_FILE src/models/bitvla.cpp:607 | if (!m->emb_reader.open(ckpt_path)) -BUGS_FILE src/models/bitvla.cpp:1074 | FILE* f = std::fopen(path.c_str(), "wb"); -BUGS_FILE src/models/bitvla.cpp:1084 | FILE* f = std::fopen(path.c_str(), "a"); -BUGS_FILE src/models/bitvla.cpp:1094 | FILE* f = std::fopen(p.c_str(), "w"); if (f) std::fclose(f); - ---- Checking src/models/vla_jepa.cpp --- -BUGS_FILE src/models/vla_jepa.cpp:227 | if (!g.open(ckpt_path)) -BUGS_FILE src/models/vla_jepa.cpp:298 | if (!io.open(gguf_path)) { -BUGS_FILE src/models/vla_jepa.cpp:299 | std::fprintf(stderr, "vla(vla_jepa): build_caches: io.open(%s) failed\n", gguf_path.c_str()); -BUGS_FILE src/models/vla_jepa.cpp:337 | FILE * fp = std::fopen(path, "wb"); if (fp) { -BUGS_FILE src/models/vla_jepa.cpp:356 | FILE * fp = std::fopen(cond_file, "rb"); -BUGS_FILE src/models/vla_jepa.cpp:376 | FILE * fp = std::fopen(patches_file, "rb"); -BUGS_FILE src/models/vla_jepa.cpp:438 | if (dump_prefix) { char nm[32]; std::snprintf(nm, sizeof(nm), "vit_view%lld", (long long) v); char path[1024]; std::snprintf(path, sizeof(path), "%s_%s_%lldx%lld.f32", dump_prefix, nm, (long long) H, (long long) K); FILE * fp = std::fopen(path, "wb"); if (fp) { std::fwrite(img_emb_host.data()+v * K * H, sizeof(float), (size_t) K * H, fp); std::fclose(fp); } } -PERF_LOOP src/models/vla_jepa.cpp:464 | { int64_t k = 0; for (int64_t p=0; pio.open(ckpt_path)) - ---- Checking src/models/evo1.cpp --- -BUGS_FILE src/models/evo1.cpp:371 | if (!m->io.open(ckpt_path)) - ---- Checking src/models/vla_adapter.cpp --- -BUGS_FILE src/models/vla_adapter.cpp:191 | if (!g.open(ckpt_path)) - ---- Checking src/models/pi0.cpp --- -BUGS_FILE src/models/pi0.cpp:359 | if (!m->io.open(ckpt_path)) - ---- Checking src/models/gr00tn1d7.cpp --- -BUGS_FILE src/models/gr00tn1d7.cpp:251 | if (!g.open(ckpt_path)) -BUGS_FILE src/models/gr00tn1d7.cpp:323 | if (!io.open(gguf_path)) { -BUGS_FILE src/models/gr00tn1d7.cpp:324 | std::fprintf(stderr, "vla(gr00tn1d7): build_caches: io.open(%s) failed\n", gguf_path.c_str()); -BUGS_FILE src/models/gr00tn1d7.cpp:719 | FILE * fp = std::fopen(path, "wb"); -BUGS_FILE src/models/gr00tn1d7.cpp:744 | FILE * fp = std::fopen(path, "wb"); - ---- Checking src/models/gr00tn1d5.cpp --- -BUGS_FILE src/models/gr00tn1d5.cpp:216 | if (!m->io.open(ckpt_path)) - ---- Checking src/models/smolvla.cpp --- -BUGS_FILE src/models/smolvla.cpp:64 | bool open(const std::string & path) { -BUGS_FILE src/models/smolvla.cpp:65 | file.open(path, std::ios::binary); -BUGS_FILE src/models/smolvla.cpp:157 | bool open(const std::string & path) { -BUGS_FILE src/models/smolvla.cpp:161 | gctx = gguf_init_from_file(path.c_str(), p); -BUGS_FILE src/models/smolvla.cpp:163 | std::fprintf(stderr, "vla: gguf_init_from_file failed for %s\n", path.c_str()); -BUGS_FILE src/models/smolvla.cpp:166 | fp = std::fopen(path.c_str(), "rb"); -BUGS_FILE src/models/smolvla.cpp:168 | std::fprintf(stderr, "vla: fopen failed for %s\n", path.c_str()); -BUGS_FILE src/models/smolvla.cpp:527 | if (!st.open(sf_path)) { -BUGS_FILE src/models/smolvla.cpp:952 | if (!gst.open(ckpt_path)) { -BUGS_FILE src/models/smolvla.cpp:1010 | if (!st.open(ckpt_path)) { - ---- Checking src/models/openvla_oft.cpp --- -BUGS_FILE src/models/openvla_oft.cpp:165 | if (!g.open(ckpt_path)) - ---- Checking src/models/pi05.cpp --- -BUGS_FILE src/models/pi05.cpp:399 | if (!m->io.open(ckpt_path)) - ---- Checking src/modules/encoder.cpp --- - ---- Checking src/modules/dit_head.cpp --- - ---- Checking src/modules/siglip_vit.cpp --- - ---- Checking src/modules/action_expert.cpp --- - ---- Checking src/modules/qwen3_lm.cpp --- - ---- Checking src/modules/prompt.cpp --- - ---- Checking src/act_dtype.h --- - ---- Checking src/arch.h --- -BUGS_MEM src/arch.h:21 | * architecture means: extend the @ref vla::Arch enum, declare a new factory - ---- Checking src/env_flag.h --- - ---- Checking src/backend.h --- - ---- Checking src/gguf_reader.h --- -BUGS_FILE src/gguf_reader.h:52 | bool open(const std::string & path) { -BUGS_FILE src/gguf_reader.h:56 | gctx = gguf_init_from_file(path.c_str(), p); -BUGS_FILE src/gguf_reader.h:58 | std::fprintf(stderr, "vla(%s): gguf_init_from_file failed for %s\n", arch, path.c_str()); -BUGS_FILE src/gguf_reader.h:61 | fp = std::fopen(path.c_str(), "rb"); -BUGS_FILE src/gguf_reader.h:63 | std::fprintf(stderr, "vla(%s): fopen failed for %s\n", arch, path.c_str()); - ---- Checking src/model.h --- - ---- Checking src/options.h --- - ---- Checking src/loader.h --- - ---- Checking src/scratch_ctx.h --- - ---- Checking src/vlm/engine.h --- - ---- Checking src/serving/hf_fetch.h --- - ---- Checking src/models/dit_common.h --- - ---- Checking src/cuda/vla_cuda_ops.h --- - ---- Checking src/kernels/bitvla/bitvla_fp32head_cuda.h --- - ---- Checking src/kernels/bitvla/bitnet_kernels.h --- -API_HEADER src/kernels/bitvla/bitnet_kernels.h:1 | Missing pragma once - ---- Checking src/kernels/bitvla/bitvla_vit_cuda.h --- - ---- Checking src/kernels/bitvla/bitvla_lm_cuda.h --- - ---- Checking src/modules/encoder.h --- - ---- Checking src/modules/dit_head.h --- - ---- Checking src/modules/qwen3_lm.h --- - ---- Checking src/modules/dual_tower.h --- - ---- Checking src/modules/gemma_expert.h --- - ---- Checking src/modules/action_expert.h --- - ---- Checking src/modules/siglip_vit.h --- - ---- Checking src/modules/prompt.h --- - ---- Checking src/modules/preprocess.h --- - ---- Checking src/modules/qwen3vl_vit.h --- - ---- Checking src/layers/norm.h --- - ---- Checking src/layers/linear.h --- - ---- Checking src/layers/attn.h --- - ---- Checking src/layers/embed.h --- - ---- Checking src/layers/ffn.h --- - ---- Checking src/layers/rope.h --- diff --git a/scanner.py b/scanner.py deleted file mode 100644 index 0b7a7d7..0000000 --- a/scanner.py +++ /dev/null @@ -1,28 +0,0 @@ -import os, glob - -def check_file(path): - with open(path, 'r') as f: - lines = f.readlines() - - print(f"\n--- Checking {path} ---") - for i, line in enumerate(lines): - line_num = i + 1 - # BUGS - if "gguf_init_from_file" in line or "fopen" in line or "open(" in line: - print(f"BUGS_FILE {path}:{line_num} | {line.strip()}") - if "malloc" in line or "new " in line: - if not "delete" in "".join(lines) and not "free" in "".join(lines): - print(f"BUGS_MEM {path}:{line_num} | {line.strip()}") - - # PERFORMANCE - if "for (" in line or "while (" in line: - if "get_f32" in line or "set_f32" in line or "memcpy" in line: - print(f"PERF_LOOP {path}:{line_num} | {line.strip()}") - - # API/BUILD - if "pragma once" not in "".join(lines) and path.endswith(".h"): - print(f"API_HEADER {path}:{line_num} | Missing pragma once") - break - -for f in glob.glob("src/**/*.cpp", recursive=True) + glob.glob("src/**/*.h", recursive=True): - check_file(f)