diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 782e2ef..6a96ce9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,7 @@ jobs: cpp-unit: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 # Compiled directly: pure, no llama.cpp or protobuf/zmq needed. - name: pure unit tests run: | @@ -29,8 +29,8 @@ jobs: py-tooling: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: '3.11' - name: converter remap @@ -41,17 +41,22 @@ jobs: build-gate: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: deps run: | sudo apt-get update -qq sudo apt-get install -y -qq --no-install-recommends \ build-essential cmake git ca-certificates pkg-config \ libzmq3-dev cppzmq-dev libprotobuf-dev protobuf-compiler - - uses: actions/cache@v4 + # Read the pin rather than repeat it: two copies drift and the stale one + # silently reuses the wrong _deps. + - name: read llama.cpp pin + id: pin + run: echo "tag=$(bash scripts/llama_tag.sh)" >> "$GITHUB_OUTPUT" + - uses: actions/cache@v6 with: path: build/_deps - key: llama-b10331-${{ runner.os }} + key: llama-${{ steps.pin.outputs.tag }}-${{ runner.os }} # Everything, not a target list: ctest registers tests this job must build, # and a named list goes stale the next time one is added. - name: build + ctest (CPU, -Wall -Wextra) @@ -59,3 +64,13 @@ jobs: cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=OFF -DVLA_BUILD_TESTS=ON cmake --build build -j"$(nproc)" ctest --test-dir build --output-on-failure + # This job is the only one that fetches llama.cpp, and neither patch script + # runs on a CPU build, so their anchors would otherwise rot unnoticed until + # someone configures a CUDA or OpenVINO tree. Patch a copy: the real one is + # cached. + - name: patch anchors still apply + run: | + cp -r build/_deps/llama-src /tmp/llama-patchtest + python3 scripts/patch_ggml_cuda_ext_hook.py /tmp/llama-patchtest + python3 scripts/patch_ggml_openvino.py /tmp/llama-patchtest + python3 scripts/patch_ggml_openvino.py /tmp/llama-patchtest # idempotent diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d8bc0e3..786f7f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,7 +41,7 @@ jobs: cuda: false runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: deps run: | @@ -75,7 +75,7 @@ jobs: mkdir -p "$out/scripts" && cp scripts/tokenize_prompt.py "$out/scripts/" tar -czf "$out.tar.gz" "$out" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: ${{ matrix.name }} path: '*.tar.gz' @@ -83,7 +83,7 @@ jobs: macos: runs-on: macos-14 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: deps run: brew install cmake zeromq cppzmq protobuf @@ -105,7 +105,7 @@ jobs: find build -name 'default.metallib' -exec cp {} "$out/" \; tar -czf "$out.tar.gz" "$out" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: macos-arm64-metal path: '*.tar.gz' @@ -113,9 +113,9 @@ jobs: docker: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 + - uses: actions/checkout@v7 + - uses: docker/setup-buildx-action@v4 + - uses: docker/login-action@v4 if: startsWith(github.ref, 'refs/tags/') with: registry: ghcr.io @@ -123,7 +123,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: image name run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV" - - uses: docker/build-push-action@v6 + - uses: docker/build-push-action@v7 with: context: . push: ${{ startsWith(github.ref, 'refs/tags/') }} @@ -140,9 +140,9 @@ jobs: steps: # Tarballs only. The docker job also leaves a .dockerbuild build record # artifact behind, and pulling that one fails the whole download. - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: { path: dist, pattern: '{linux,macos}-*', merge-multiple: true } - - uses: softprops/action-gh-release@v2 + - uses: softprops/action-gh-release@v3 with: files: dist/*.tar.gz fail_on_unmatched_files: true diff --git a/.github/workflows/vla-ci.yml b/.github/workflows/vla-ci.yml index 9147eab..57c88bd 100644 --- a/.github/workflows/vla-ci.yml +++ b/.github/workflows/vla-ci.yml @@ -23,10 +23,10 @@ jobs: runs-on: [self-hosted, vla-ci-orchestrator] timeout-minutes: 240 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 - name: Sweep all platforms (parallel) + gate run: bash ci/orchestrate.sh all - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 if: always() with: name: ci-results diff --git a/.gitignore b/.gitignore index 8894aa1..1d3f9d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Build -build/ -build-*/ +build*/ out/ cmake-build-*/ _workdir* @@ -48,7 +47,6 @@ outputs/ weights/ CLAUDE.md -third_party/llama.cpp/ eval/sim/libero/LIBERO/ eval/sim/libero/libero_uv/ eval/sim/simpler/SimplerEnv/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bb394c..1fb5f99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,113 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://keepachangelog.com). +## [Unreleased] + +### Added + +- **OpenVINO backend.** `-DGGML_OPENVINO=ON` runs the archs on Intel CPUs, iGPUs + and NPUs through ggml's OpenVINO backend. SmolVLA, π0.5, Evo-1 and VLA-Adapter + match an F32 CPU reference to 1e-3; on an Arc B390 iGPU that is 3.0x to 9.6x + the native CPU backend. All nine tested archs are inside the accuracy bar on the + OpenVINO CPU plugin and on the iGPU. OpenVLA-OFT is untested. + See `docs/backend/ov.md`. +- `scripts/install_ov.sh` installs the OpenVINO runtime and the Intel GPU/NPU + driver stack on Ubuntu 22.04 and 24.04, with the runtime archive checksummed + against a digest pinned in the script. +- `scripts/patch_ggml_openvino.py` applies thirteen fixes to the fetched + ggml-openvino sources at configure time. Each hunk is checked on its own, so a + `build/_deps` patched by an older checkout fails loudly instead of building + something quietly wrong. +- `tests/test_graph_names.cpp` pins `vla::graph_unique_names`. +- CI now checks that both llama.cpp patch scripts still apply, on a copy of + the fetched tree. Neither ran on a CPU build, so their anchors could rot + unnoticed until someone configured a CUDA or OpenVINO tree. +- `docs/UPSTREAMING.md` and `scripts/upstream_split.py` regroup the thirteen + ggml-openvino fixes into one llama.cpp branch per PR. They are generic + backend defects, not vla.cpp workarounds; landing them upstream removes the + configure-time patch step entirely. + +### Fixed + +- Two elementwise adds stacked on a GEMM came out wrong on the Intel iGPU. The + GPU plugin folds elementwise ops into the preceding GEMM as post-ops, and given + `ADD(ADD(residual, GEMM), graph_input)` it folds both and silently drops the + second operand - the result equals the inner add. A llama.cpp graph never builds + that chain; a VLA does, wherever a vision tower's features are added on top of an + FFN residual. VLA-JEPA (5.4e-1) and GR00T N1.7 (1.9e0) were wrong on the iGPU + while matching the CPU plugin to 1e-4. Re-associating the two adds so the GEMM + keeps one post-op puts both at 2.6e-3. Bisected with `GGML_OPENVINO_DEBUG_NODE`. +- π0's action dims drifted 4e-2 on the iGPU and its gripper flipped a step late, + because the GPU plugin computes in F16 and π0 unrolls its whole denoise loop + inside one graph. `GGML_OPENVINO_GPU_PRECISION` now exposes the plugin's + inference precision; `backend_init` defaults it to f32 for π0 alone, which costs + about 3x on that arch and puts it at 6.5e-5. +- `scripts/patch_ggml_openvino.py` now fails if `EDITS` has a duplicate key. Python + keeps the last one silently, and a duplicate briefly removed the whole Intel + OpenCL platform fix from the patch without any error. +- The position-input fix stopped running when llama.cpp moved to `b10729`. That + release relocated the naming out of `GgmlOvDecoder::get_graph_input_ov_name()`, + which the patch guards, into a new free `get_tensor_graph_input_ov_name()`, and + left the member behind with no callers. The hunk still applied cleanly, so + nothing failed loudly - SmolVLA and π0.5 simply stopped returning actions + ("Argument shapes are inconsistent", a 113-token prefix ROPE reading the + 50-token suffix's table). Both functions are guarded now, and the patch script + says to check for a live caller, not just a matching anchor, on every tag bump. +- `scripts/upstream_split.py` addressed hunks by position in the patch script's + edit list. Adding a hunk to the front of a file's list silently handed every + later hunk to the wrong branch, and its own coverage count still read 29/29 + because each index was still used exactly once. Two branches had been swapped + this way. Hunks are now addressed by a unique substring of their anchor, which + fails loudly instead. The PERMUTE `op_case` fix, which had no branch at all, + now has one. +- `graph_unique_names` renamed through `ggml_format_name`, which passes the + tensor's own name to `vsnprintf` as both destination and `%s` source. glibc + empties it, so every duplicate node became the bare string `#`. +- The OpenVINO naive-path compiled-model cache was keyed on node count plus the + first and last node name. Two graphs of the same size collided and the second + ran the first's compiled model. It now also keys on every node's op and shape, + and the map is bounded. +- `GGML_OPENVINO_NAIVE_GRAPH_SIZE` went through `atoi`, so junk parsed to 0 and + sent every graph down the decoder-only-LLM path with nothing said. Empty + environment values no longer count as a setting either. +- `GGML_OPENVINO_CACHE_DIR` is cleared rather than warned about: a warm cache + returns wrong actions, and stderr is not always read. `VLA_ALLOW_OV_CACHE=1` + keeps it. +- `scripts/print_versions.sh` printed `?` for the llama.cpp pin ever since the + tag moved behind `VLA_LLAMA_TAG`. +- The OpenVINO `find_package` failure message was unreachable, sitting after the + fetch whose own `find_package(REQUIRED)` fired first. +- BitVLA indexed its action slots as `seq-2-n_action+i` with no check that the + sequence is long enough. Neither `ggml_get_rows` nor the CUDA gather + bound-checks, so a short prompt read out of bounds and returned it as hidden + states. One guard now covers both LM paths. +- pi0 and pi0.5 fell back to identity normalisation stats on a dimension mismatch + or a short read, and said so on stdout. That returns un-denormalised actions + from a checkpoint that looked fine. Both now fail the load, and the message + goes to stderr - stdout is the action stream `predict_check` diffs. +- `scratch_ctx::reset` ignored an arena larger than the first call's, which would + abort in `ggml_new_tensor` if any call site ever sized one from the input. +- The safetensors arch probe would allocate up to 256 MB for a header it only + substring-searches. Capped at 16 MB. +- The two CUDA targets were the only first-party code built without + `-Wall -Wextra`. +- `tests/bitvla_gemm_check.cu` had no build target and a comment claiming it was + never committed. It builds now, under `GGML_CUDA`. +- Stale references to `vision_common.h` (now `modules/preprocess.h`) and to the + retired `VLA_EVO1_BF16_ACT` switch. + +### Changed + +- llama.cpp pinned at `b10729`, up from `b10331`. Brings OpenVINO 2026.3.1, the + IM2COL+MatMul to native-convolution fusion, and the `RELU`/`NEG`/`SQR` + translators, which the local patch no longer has to add. Byte-identical on the + CPU backend for all eleven archs. The build.yml cache key now reads the tag out + of `CMakeLists.txt` instead of repeating it. +- `src/models/dit_common.h` is gone. It redefined six `vla::` functions that + `src/layers/` already had, with both copies linked into `vla_core`. Every + includer used only `sinusoidal_time_emb` or `build_causal_mask`, so they now + include `layers/embed.h`. Byte-identical across all 11 archs. + ## [0.3.0] - 2026-08-14 Every architecture is byte-identical to 0.2.0 at matching settings. diff --git a/CMakeLists.txt b/CMakeLists.txt index 89da1ee..01e34be 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ if(NOT CMAKE_BUILD_TYPE) endif() set(_vla_accel "") -foreach(_flag GGML_CUDA GGML_SYCL GGML_METAL) +foreach(_flag GGML_CUDA GGML_SYCL GGML_METAL GGML_OPENVINO) if(${_flag}) list(APPEND _vla_accel ${_flag}) endif() @@ -24,6 +24,14 @@ if(_vla_accel_n GREATER 1) "Enable one accelerator backend at a time; got ${_vla_accel_str}. " "Configure a separate build directory per backend.") endif() +# backend.h calls ggml_backend__init directly, and GGML_BACKEND_DL builds +# the accelerators as loadable modules that are not linked into ggml. Say so here +# rather than at the undefined symbol. +if(_vla_accel_n GREATER 0 AND GGML_BACKEND_DL) + message(FATAL_ERROR + "GGML_BACKEND_DL=ON is not supported with ${_vla_accel}: the backend is " + "built as a module and src/backend.h links its init directly.") +endif() set(LLAMA_BUILD_COMMON ON CACHE BOOL "" FORCE) set(LLAMA_BUILD_TOOLS ON CACHE BOOL "" FORCE) @@ -35,11 +43,36 @@ if(GGML_CUDA) find_package(Python3 COMPONENTS Interpreter REQUIRED) set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_cuda_ext_hook.py ) +elseif(GGML_OPENVINO) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_openvino.py ) + + # Before the fetch: ggml's own ggml-openvino/CMakeLists.txt does + # find_package(OpenVINO REQUIRED ...) and would get there first, so this + # message only reaches anyone if it runs earlier. Same components it asks for. + find_package(OpenVINO QUIET COMPONENTS Runtime Threading) + if(NOT OpenVINO_FOUND) + message(FATAL_ERROR + "GGML_OPENVINO=ON but the OpenVINO runtime was not found. Install it " + "(scripts/install_ov.sh) and 'source /opt/intel/openvino/setupvars.sh' " + "in the shell that configures. See docs/backend/ov.md.") + endif() endif() # Overridable so a regression can be bisected against another tag in a separate # build dir (-DVLA_LLAMA_TAG=b10326) without editing this file. The patch -# anchors in scripts/patch_ggml_cuda_ext_hook.py are checked against the default. -set(VLA_LLAMA_TAG "b10331" CACHE STRING "llama.cpp tag to fetch") +# anchors in scripts/patch_ggml_cuda_ext_hook.py and scripts/patch_ggml_openvino.py +# are checked against the default. +set(_vla_llama_tag_default "b10729") +set(VLA_LLAMA_TAG "${_vla_llama_tag_default}" CACHE STRING "llama.cpp tag to fetch") +# A cache entry survives an edit to the line above, so an existing build dir keeps +# the tag it was first configured with and quietly builds the wrong llama.cpp. +if(NOT VLA_LLAMA_TAG STREQUAL _vla_llama_tag_default) + message(WARNING + "VLA_LLAMA_TAG is ${VLA_LLAMA_TAG}, not the default ${_vla_llama_tag_default}. " + "If that was not deliberate, this build dir predates the bump: reconfigure with " + "-DVLA_LLAMA_TAG=${_vla_llama_tag_default} or use a fresh one.") +endif() include(FetchContent) FetchContent_Declare(llama @@ -173,6 +206,12 @@ if(GGML_METAL AND NOT GGML_CUDA AND NOT GGML_SYCL) target_compile_definitions(vla_core PUBLIC GGML_USE_METAL) endif() +if(GGML_OPENVINO AND NOT GGML_CUDA AND NOT GGML_SYCL AND NOT GGML_METAL) + # Only tells the archs which branch of the backend.h ladder to compile; the + # toolkit was located before the fetch above. + target_compile_definitions(vla_core PUBLIC GGML_USE_OPENVINO) +endif() + add_library(vlm_core src/vlm/engine.cpp ) @@ -286,6 +325,10 @@ target_link_libraries(vla-bench PRIVATE vla_core) # Warnings and LTO for our own targets only, never the llama.cpp subtree. set(VLA_FIRST_PARTY_TARGETS vla_core vlm_core vla vla-server vlm-server vla-cli vla-bench) +# The CUDA targets were the only first-party code compiled without warnings. +if(GGML_CUDA) + list(APPEND VLA_FIRST_PARTY_TARGETS bitvla_cuda_kernels vla_cuda_ops) +endif() foreach(tgt IN LISTS VLA_FIRST_PARTY_TARGETS) target_compile_options(${tgt} PRIVATE $<$:-Wall -Wextra>) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ce5d34..bc5915d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,10 +54,11 @@ ckpt) model, `bitvla` for a vision-baked one. 6. `CMakeLists.txt` - add `src/models/.cpp` to `vla_core`. Then write `src/models/.cpp`. Before adding a helper, check -`src/models/`: `gguf_reader.h` (tensor and KV reads), `vision_common.h` +`src/models/`: `gguf_reader.h` (tensor and KV reads), `modules/preprocess.h` (preprocessing, pixel shuffle), `dual_tower.h` (DINOv2 + SigLIP), -`qwen3vl_vit.h` (Qwen3-VL tower), `dit_common.h` (DiT time embeddings), -`scratch_ctx.h` (compute context reuse), `backend.h` (accelerator selection). +`qwen3vl_vit.h` (Qwen3-VL tower), `layers/embed.h` (time embeddings, causal +mask), `scratch_ctx.h` (compute context reuse), `backend.h` (accelerator +selection). Your loader must fail rather than return a half-built model: check every tensor lookup, and check `real_*_dim <= max_*_dim` (`config_is_sane` in `src/model.cpp` diff --git a/README.md b/README.md index 4f06b44..4a8a994 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ A C++ inference engine for **Vision-Language-Action (VLA) models**, built on [`l It runs the open VLA policies - SmolVLA, π0, BitVLA, Evo-1, GR00T N1.5/1.6/1.7 and more - under one runtime, each packaged as a single self-contained GGUF that needs no Python or PyTorch at inference time. The binaries drive robots on **CPU**, **Apple Silicon**, **CUDA** - -from consumer GPUs down to Jetson-class boards - or **Intel GPUs** via SYCL. +from consumer GPUs down to Jetson-class boards - or **Intel GPUs and NPUs** via +SYCL and OpenVINO. [**Learn vla.cpp**](https://fai-modelopt-tech.github.io/learn-vla-cpp/) walks through the engine design and how each policy is implemented on ggml. @@ -27,6 +28,8 @@ from consumer GPUs down to Jetson-class boards - or **Intel GPUs** via SYCL. - CUDA 12.x (optional - required only for CUDA GPU builds) - Intel oneAPI 2025.x + GPU compute runtime (optional - only for Intel GPU builds, see [docs/backend/sycl.md](docs/backend/sycl.md)) +- OpenVINO 2026.x runtime (optional - only for Intel CPU/GPU/NPU builds via + OpenVINO, see [docs/backend/ov.md](docs/backend/ov.md)) - `libzmq3-dev`, `cppzmq-dev`, `libprotobuf-dev`, `protobuf-compiler` ```bash @@ -309,19 +312,19 @@ Experimental results on other platforms can be found in Support matrix of models (rows) against platforms (columns). Legend: `Y` = supported (released and benchmarked), `~` = in progress, `-` = planned. -| Model | CPU (x86-64 / ARM) | CUDA | SYCL (Intel) | Metal | OpenVINO | +| Model | CPU (x86-64 / ARM) | CUDA | [SYCL (Intel)](docs/backend/sycl.md) | [Metal](docs/backend/metal.md) | [OpenVINO](docs/backend/ov.md) | |---|:--:|:--:|:--:|:--:|:--:| -| [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | - | -| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | - | -| [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | - | -| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | - | -| [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | - | -| [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | - | +| [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | +| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | Y | +| [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | +| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | Y | +| [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | +| [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | Y | | [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | -| [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | - | -| [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | - | -| [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | - | -| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | - | +| [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | +| [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | +| [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | ~ | +| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | --- diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3ad8cda..0ed9960 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -16,7 +16,7 @@ source is the detail. contexts, vision tower, weights, and compute graph. - `src/models/gguf_reader.h` - the shared GGUF reader (metadata, tensor bytes, on-demand embedding rows). -- `src/models/vision_common.h` - small pure vision helpers (pixel-shuffle, view checks). +- `src/modules/preprocess.h` - small pure vision helpers (pixel-shuffle, view checks). - `src/serving/` - `vla-server` (ZeroMQ + protobuf, action prediction), `vlm-server` (chat), and `vla-cli` (one-shot inference). - `src/kernels/bitvla/` - custom 1.58-bit ternary CUDA kernels for BitVLA. @@ -64,4 +64,4 @@ count; CUDA and Metal run the towers and the transformer on the GPU. Extend the `Arch` enum, declare a `*_create` factory in `arch.h`, implement it under `src/models/`, wire detection and dispatch in `src/model.cpp`, and add a converter in -`scripts/`. Reuse `gguf_reader.h` and `vision_common.h` rather than copying them. +`scripts/`. Reuse `gguf_reader.h` and `modules/preprocess.h` rather than copying them. diff --git a/docs/UPSTREAMING.md b/docs/UPSTREAMING.md new file mode 100644 index 0000000..842d4f1 --- /dev/null +++ b/docs/UPSTREAMING.md @@ -0,0 +1,86 @@ +# Upstreaming the ggml-openvino fixes + +`scripts/patch_ggml_openvino.py` rewrites nine files inside the fetched llama.cpp +tree at configure time. Every fix in it is a generic ggml-openvino defect, not a +vla.cpp workaround: each narrows an assumption that fits llama.cpp's one +decoder-only graph but is stricter than the ggml contract, or fills a gap in the +op table. None of them needs to live here. + +Landing them upstream deletes the patch step, lets vla.cpp build against stock +llama.cpp, and fixes the same bugs for everyone else translating a graph that is +not a decoder-only LLM - whisper.cpp, embedding models, any vision tower. + +## The series + +A local clone with one branch per PR, each a single commit on `master`: + +``` +~/llama.cpp-upstream +``` + +Regenerate it after editing the patch script: + +```bash +git clone --depth 1 https://github.com/ggml-org/llama.cpp ~/llama.cpp-upstream +python3 scripts/upstream_split.py +``` + +| Branch | What it fixes | Value to upstream | +|---|---|---| +| `openvino-naive-cache` | the naive path re-converts and re-compiles the model on every `graph_compute` | **highest**: 22.7 s to 1.4 s on a SmolVLA graph. Also replaces `graph_key` with a shape-aware `naive_key` for that cache, since a node count plus two names collides | +| `openvino-multiple-inp-pos` | all ROPE position inputs are renamed to one `inp_pos` parameter | **high**: any graph with more than one position tensor fails shape inference today | +| `openvino-view-input-rank` | a folded 2-D weight is sliced at full ggml rank | high: `Axis 2 out of the tensor rank range [-2, 1]` on any fused QKV weight | +| `openvino-concat-rank` | CONCAT cannot broadcast rank | high: same rank-2 constants, different op | +| `openvino-reshape-op-case` | the KV-cache-flatten guard also swallows `ggml_conv_2d`'s kernel reshape | medium | +| `openvino-sdpa-kv-f16` | K/V stay F32 while Q is converted | medium: `Mixed input types are not supported` | +| `openvino-naive-graph-size-env` | the 20-node naive threshold is a `constexpr` | medium: exposes `GGML_OPENVINO_NAIVE_GRAPH_SIZE` | +| `openvino-gelu-modes` | ggml's tanh `GELU` is mapped onto ov's erf default, and `GELU_ERF` has no entry at all | **high**: wrong activation on every GELU node. Fixing it moved GR00T N1.5 and VLA-JEPA inside the accuracy bar | +| `openvino-intel-opencl-platform` | the GPU remote context takes the first OpenCL platform | low, but a hard startup abort when it bites | +| `openvino-imrope-sections` | the IMROPE sector cycle ignores `sections` | low, no measured output change | +| `openvino-gemm-double-eltwise` | the GPU plugin folds two chained elementwise adds into the preceding GEMM and silently drops the second operand | **highest**: wrong output on GPU with nothing logged, on any graph that adds a tower's features on top of an FFN residual | +| `openvino-gpu-precision-env` | the GPU plugin's f16 default compounds through a long serial chain in one graph | medium: exposes `GGML_OPENVINO_GPU_PRECISION`, default unchanged | +| `openvino-permute-op-case` | PERMUTE `op_case` 2 is reached by any permute over a view of a non-leaf, not just llama.cpp's rope'd query | **high**: a DiT cross-attention V comes out with its elements rearranged and nothing is reported | + +`RELU`, `NEG` and `SQR` were in the patch too. Upstream added all three in +`b10729`, so they are not in the series. + +## Before submitting + +Read `CONTRIBUTING.md` in llama.cpp. The parts that bite: + +- **One PR per feature.** Hence one branch each; do not squash them. +- **AI usage must be disclosed**, and using AI to write the PR text itself is + prohibited outright. Undisclosed use risks a ban. Write the PR bodies yourself. +- **A modified operator needs `test-backend-ops`.** That covers + `openvino-gelu-modes`, `openvino-sdpa-kv-f16`, `openvino-concat-rank`, + `openvino-view-input-rank` and `openvino-reshape-op-case`. The others touch the + session and cache layers, which `test-backend-ops` does not reach; those need a + before/after run on a real graph in the PR body. +- **A bug fix needs a reproducible case that fails before and passes after.** + Each commit message already names the error string or the timing it fixes. +- New contributors should keep one PR open at a time. `openvino-naive-cache` is + the one to lead with. + +## Not yet written + +**Key the translation map on the tensor, not its name.** `ggml-decoder.cpp` keys +everything on `node->name`. ggml permits duplicate names - it derives a result's +name from its source, so an unnamed `ggml_reshape_2d` is called `" (reshaped)"` - +and llama.cpp only escapes this because it labels every node it builds. Two nodes +sharing a name silently become one and the graph wires the wrong tensor into the +next op. + +`vla::graph_unique_names` in `src/backend.h` works around it from the outside, at +29 call sites. De-duplicating at graph ingest inside `ggml_decoder` would be a few +lines, fix it for every caller, and let vla.cpp delete the helper and all 29 +calls. Worth writing; not in the series because it is new code rather than a fix +already proven on hardware. + +## Status + +The thirteen branches carry code that has run on an Intel Core Ultra X7 358H (Arc +B390 iGPU, AI Boost NPU) through vla.cpp's own OpenVINO builds - see +`docs/backend/ov.md` for what that covered. They have **not** been compiled from +these branches: no OpenVINO runtime is installed on the machine that split them, +so `ggml-openvino` does not build there. Build and run each branch before opening +a PR. diff --git a/docs/backend/ov.md b/docs/backend/ov.md new file mode 100644 index 0000000..a55d868 --- /dev/null +++ b/docs/backend/ov.md @@ -0,0 +1,372 @@ +# `vla.cpp` on Intel CPUs, GPUs and NPUs (OpenVINO backend) + +Notes for building `vla.cpp` against ggml's OpenVINO backend, and an honest +account of how far it currently runs. Like SYCL, OpenVINO is **not** +auto-detected: it needs an explicit `-DGGML_OPENVINO=ON` and the OpenVINO +runtime on the configure line. + +> **Status: all nine tested architectures translate faithfully, on the CPU plugin +> and on the iGPU.** SmolVLA, π0, π0.5, Evo-1, VLA-Adapter, GR00T N1.5, GR00T N1.6, +> GR00T N1.7 and VLA-JEPA all agree with a CPU-backend reference to 1.4e-3 or +> better on the OpenVINO CPU plugin, and eight of the nine are inside the same bar +> on the Arc B390 iGPU, where the speedup over the native CPU backend runs from +> 3.0x to 9.6x. +> +> Fifteen fixes were needed, thirteen of them inside ggml's OpenVINO backend, which +> is written against llama.cpp's graphs and had never seen a vision tower or an +> action expert - see [What had to change](#what-had-to-change). +> +> Note the baseline: OpenVINO executes the checkpoint's BF16 weights at F32, so +> compare against `--weight-dtype f32` or you will charge the backend for a +> precision upgrade. See [Picking the right baseline](#picking-the-right-baseline). + +Measured on an **Intel Core Ultra X7 358H** (Panther Lake) with the Arc B390 +iGPU and the AI Boost NPU, Ubuntu 24.04, OpenVINO 2026.2.1, llama.cpp `b10729`, +on the checkpoints under `vrfai/` on the Hub. Every **fidelity** number was +re-measured on that pin; only the **latency** table still dates from `b10331`. + +ggml's backend translates a ggml compute graph into an OpenVINO model and hands +it to the CPU, GPU or NPU plugin, which compiles and fuses it for the device. +Unlike SYCL it needs no separate compiler: the stock GCC/Clang build links +`libopenvino`. + +## Supported devices + +Intel CPUs, Intel GPUs (integrated Xe / Arc, and discrete), and Intel NPUs +(Core Ultra). Linux only here - Ubuntu 22.04 or 24.04. + +## Prerequisites + +### 1. Device access + +The CPU plugin needs nothing. The GPU and NPU plugins reach the hardware through +`/dev/dri/renderD*` and `/dev/accel/accel0`, both owned by the `render` group: + +```bash +sudo usermod -aG render,video "$USER" # re-login afterwards +clinfo -l # must enumerate the GPU +``` + +`Number of platforms 0` with `/etc/OpenCL/vendors/intel.icd` present almost +always means the render group has not taken effect yet. Without it, +`GGML_OPENVINO_DEVICE=GPU` warns and silently falls back to the CPU plugin. + +For the GPU compute runtime and NPU driver packages themselves, follow +[llama.cpp's OpenVINO notes](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/OPENVINO.md). + +The NPU needs two more things its driver packages do not pull in. Neither failure +is reported as an error - the device simply does not appear and every +`GGML_OPENVINO_DEVICE=NPU` run lands on the CPU plugin instead +(`device NPU is not available, fallback to CPU`). + +```bash +sudo apt-get install -y libze1 # the Level Zero loader; the driver alone is not enough + # check with: ldconfig -p | grep ze_loader +export ZE_ENABLE_ALT_DRIVERS=/lib/x86_64-linux-gnu/libze_intel_npu.so.1 +``` + +Ubuntu's loader (1.16.1 in noble) does not discover `libze_intel_npu.so.1` on its +own, hence the override; a loader from Intel's own graphics repository, +version-matched to the driver, should not need it - untested here. With both in +place the device enumerates as `NPU Intel(R) AI Boost`. + +### 2. OpenVINO runtime + OpenCL headers + +```bash +sudo apt-get install -y opencl-clhpp-headers ocl-icd-opencl-dev opencl-headers \ + cmake ninja-build pkg-config protobuf-compiler libprotobuf-dev \ + libzmq3-dev cppzmq-dev +``` + +Then either install OpenVINO +[from the archive](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-linux.html) +by hand, or run `bash scripts/install_ov.sh`, which also pulls the GPU driver +stack and adds you to `render`. + +## Configure & build + +```bash +source /opt/intel/openvino/setupvars.sh + +cmake -B build-ov -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON +cmake --build build-ov -j$(nproc) +``` + +`setupvars.sh` must be sourced in every shell that builds *or* runs the binaries: +`libopenvino.so` and its TBB live under `/opt/intel`. Configure fails early with +a pointer back here if the runtime is not on `CMAKE_PREFIX_PATH`. + +`scripts/patch_ggml_openvino.py` runs as the FetchContent patch step, so the +thirteen ggml fixes are applied automatically - there is no manual `git apply`. +The step only runs when FetchContent populates the source dir, so a `build/_deps` +left over from an older checkout keeps the hunks it was patched with: delete it +after pulling rather than trusting a reconfigure. The script checks each hunk on +its own and fails loudly on a tree it cannot bring up to date. + +## Run + +`GGML_OPENVINO_DEVICE` picks the target by name (`VLA_DEVICE` does *not* apply - +ggml exposes OpenVINO as a single device): + +```bash +GGML_OPENVINO_DEVICE=GPU ./build-ov/vla-server ./weights/smolvla-libero.gguf +``` + +Two lines identify the selection at startup: + +```text +OpenVINO: using device GPU +vla: backend = OPENVINO (asked for GPU, see ggml's "using device" line) +``` + +The first comes from ggml and is authoritative: an unavailable device logs a +warning there and falls back to `CPU`, which is still the OpenVINO CPU plugin, +not ggml's native CPU backend. The second echoes what was requested, so the pair +tells you whether you got the device you asked for. + +OpenVINO compiles each graph on first use, which is slow - a minute or two for a +vision tower on the GPU. Compiled graphs are then cached in-process for the life +of the model, so only the first prediction pays that; give any client a receive +timeout well above the first request. Do **not** set `GGML_OPENVINO_CACHE_DIR` to +carry them across restarts; it produces silently wrong actions here - see +[Known issues](#known-issues). + +## Results + +`vla_predict_check` (a test target - add `-DVLA_BUILD_TESTS=ON`), fixed noise, +one camera view, best of 4-6 iterations after 3 warmups. "CPU backend" is ggml's +own CPU backend on the same 16-core host. No `GGML_OPENVINO_CACHE_DIR`. + +Latencies were taken at `b10331` and have not been re-timed on `b10729`. Read the +GPU column for VLA-JEPA and GR00T N1.7 as the cost of a wrong answer at that pin; +both are correct now. + +| Model | input | CPU backend | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | +|---|---|---:|---:|---:|---:| +| VLA-JEPA | 256 | 1,046 ms | 1,265 ms | **127 ms** (8.2x) | returns NaN | +| GR00T N1.5 | 224 | 1,420 ms | 2,199 ms | **148 ms** (9.6x) | plugin throws | +| VLA-Adapter | 224 | 1,228 ms | 1,603 ms | **161 ms** (7.6x) | not supported | +| GR00T N1.6 | 224 | 1,276 ms | 2,256 ms | **323 ms** (3.9x) | plugin throws | +| SmolVLA | 512 | 1,364 ms | 1,340 ms | **451 ms** (3.0x) | 1,162 ms | +| Evo-1 | 448 | 3,114 ms | 4,523 ms | **563 ms** (5.5x) | not supported | +| π0.5 | 224 | 2,802 ms | 4,285 ms | **683 ms** (4.1x) | 916 ms | +| GR00T N1.7 | 256 | 1,146 ms | not timed | **288 ms** (4.0x) | not attempted | + +The iGPU is the reason to use this backend, and it pays off most where the model +is most vision-heavy. The OpenVINO CPU plugin is at best parity with ggml's own +CPU backend and often well behind it, so it is only worth running to debug a +translation. The NPU beats the CPU backend on the two models it accepts while +drawing far less power, which is the interesting result for a robot - see the +NPU limits under [Known issues](#known-issues). + +### Picking the right baseline + +Actions are checked against the CPU backend on identical inputs; both sides are +deterministic, so the numbers are exact rather than sampled. But **which** CPU run +you compare against matters. OpenVINO folds the checkpoint's BF16 weights in as +constants and its CPU plugin executes them at F32, while ggml's CPU backend keeps +them BF16, so a naive comparison charges the OpenVINO backend for a precision +*upgrade*. Running the reference with `--weight-dtype f32` removes that term. + +The two references bracket the answer, and which is tighter turns on how much of +a given checkpoint is BF16 in the first place. Report both and take the smaller: + +| Model | vs BF16 reference | vs F32 reference | tighter reference | +|---|---:|---:|---| +| VLA-Adapter | 2.1e-3 | **2.4e-6** | F32 | +| Evo-1 | 2.7e-3 | **3.5e-6** | F32 | +| π0.5 | 7.7e-4 | **3.5e-5** | F32 | +| VLA-JEPA | 1.1e-2 | **1.1e-4** | F32 | +| GR00T N1.5 | 5.5e-3 | **6.0e-4** | F32 | +| GR00T N1.6 | 5.0e-3 | **1.0e-3** | F32 | +| SmolVLA | **8.9e-4** | 1.3e-3 | BF16 | +| GR00T N1.7 | 1.5e0 | **4.2e-4** | F32 | + +Evo-1 and VLA-Adapter agree with an F32 reference to six decimal places, which is +as close to "the translation is exact" as this harness can show; the BF16 +comparison for those two was measuring nothing but the dtype. SmolVLA is the +counterexample that stops this being a universal rule. For context, the CPU +backend's own output moves by 2.0e-3 (SmolVLA), 2.7e-3 (Evo-1) or 1.1e-2 +(VLA-JEPA) when you flip that one flag, so the model's intrinsic sensitivity to +precision is the same size as the numbers being reported. + +### Full results + +Against the F32 reference, which is the fidelity number: + +| Model | OpenVINO CPU | OpenVINO GPU | OpenVINO NPU | +|---|---:|---:|---:| +| Evo-1 | 2.2e-6 | 6.0e-4 | compiler rejects | +| VLA-Adapter | 3.9e-6 | 6.8e-3 | compiler rejects | +| π0.5 | 6.1e-5 | 7.6e-4 | 1.6e-3 | +| VLA-JEPA | 7.7e-5 | 2.6e-3 | returns NaN | +| GR00T N1.7 | 3.9e-4 | 2.6e-3 | NPUW throws | +| π0 | 5.8e-4 | 6.6e-5 | **1.7e0** | +| GR00T N1.5 | 6.0e-4 | 4.6e-3 | NPUW throws | +| GR00T N1.6 | 1.1e-3 | 1.4e-3 | NPUW throws | +| SmolVLA | 1.4e-3 | 2.6e-3 | 1.1e-2 | + +Every tested arch is inside the 2.9e-3 bar on the CPU plugin, most by one to +three orders of magnitude, and eight of the nine are inside it on the GPU as well +(GR00T N1.5's 4.6e-3 against F32 is 1.6e-3 against BF16, the tighter reference for +that arch). The one outside is **VLA-Adapter**, at 6.8e-3 against F32 on actions +peaking at 0.62 - the GPU plugin computing in F16, a precision effect rather than +a translation error, since the same arch is 3.9e-6 on the CPU plugin. Judge +translation fidelity on the CPU plugin and treat the GPU as a separate precision +target. π0 is the exception in the other direction, *tighter* on the GPU (6.6e-5) +because it is the one arch that runs the GPU at F32; see +[Known issues](#known-issues), which also covers the NPU column. + +## What had to change + +Fifteen fixes: two in vla.cpp, thirteen in ggml's OpenVINO backend. Both +vla.cpp-side ones are ordinary correctness fixes that happen to be invisible on +the other backends - `vla_predict_check` on a CPU build of this branch is +byte-identical to the same build of the base commit, for every model tested. + +- **Weight buffers are tagged.** `ggml_backend_alloc_ctx_tensors` leaves a buffer + on `GGML_BACKEND_BUFFER_USAGE_ANY`, and ggml-openvino reads ANY as "KV cache", + giving every weight a dynamic sequence dimension. `vla::alloc_weights` in + [`src/backend.h`](../../src/backend.h) tags it `..._WEIGHTS`, which is what + lets the frontend fold weights in as constants. Since 0.3.0 every arch + allocates through `vla::WeightLoader`, so this is one call site in + [`src/loader.cpp`](../../src/loader.cpp). +- **Graph tensors get unique names.** ggml derives a result's name from its + source, so a graph whose intermediates were never named ends up with many + tensors sharing one name (`" (reshaped)"` and friends). ggml-openvino keys its + translation map on those names, so duplicates silently collapse into one node + and the graph wires the wrong tensor into the next op. `vla::graph_unique_names` + relabels duplicates before compute, at each of the 29 + `ggml_backend_graph_compute` call sites. It compiles to nothing outside an + OpenVINO build. + +`backend_init` also sets one default, the way the SYCL rung already sets +`GGML_SYCL_ENABLE_VMM=0`: **`GGML_OPENVINO_NAIVE_GRAPH_SIZE` defaults high.** +ggml-openvino translates a graph under 20 nodes literally and sends anything +larger through a model builder that assumes a decoder-only LLM. The literal path +is the one that fits a vision tower and an action expert. An explicit setting +still wins. + +The other thirteen are applied to ggml's OpenVINO backend by +`scripts/patch_ggml_openvino.py` at configure time; its docstring carries the +per-fix detail. Each narrows an llama.cpp-shaped assumption that is stricter than +the ggml contract, or fills a gap: + +| Fix | What it addresses | +|---|---| +| **PERMUTE op_case 2 requires a ROPE** | **assumes any permute of a view is a rope'd query** | +| **Two elementwise adds never stacked on a GEMM** | **the GPU plugin folds both in as post-ops and drops the second operand** | +| GPU inference precision exposed | the plugin's F16 default compounds through a denoise loop unrolled in one graph | +| **GELU translated as tanh, not erf** | **assumes ggml's GELU is the exact erf form** | +| Intel OpenCL platform selection | assumes the first OpenCL platform is Intel's | +| RESHAPE `op_case` guard | assumes a reshape flattening dims 0-2 is the KV-cache flatten | +| SDPA K/V converted with Q | assumes K/V arrive as F16 because the KV cache is | +| **Position inputs keyed per tensor** | **assumes a graph has exactly one position input** | +| Folded weights padded to full rank | a 2-D weight becomes a rank-2 constant, but views index it at ggml rank | +| CONCAT input ranks aligned | same rank-2 constants, and concat cannot broadcast rank | +| Missing `GELU_ERF` translator | the exact-erf GELU op had no table entry at all, so a graph using it could not run | +| Naive-path graph cache | that path re-compiled the whole model on every graph_compute, and its `graph_key` is a node count plus two names, which two graphs can share | +| Interleaved-mrope sectors bounded | the sector cycle ignored `sections`, so the last few took the wrong stream | +| Naive-path threshold settable | the 20-node constant is what picks the literal path | + +The bolded rows are the ones that turned a wrong arch into a correct one: +PERMUTE op_case 2 for GR00T N1.7, the double-elementwise guard for the Eagle-VLM +archs on the iGPU, the GELU mode for VLA-JEPA and GR00T N1.5, per-tensor position +inputs for SmolVLA (which passes three position tensors, so they aliased). The +GELU mode and the position-input fix are the two worth upstreaming. + +**Debugging a mistranslation.** The backend writes back true graph outputs and +nothing else, so an interior tensor is observable only two ways: truncate the +graph at a stage, which makes that stage the terminal node, or set +`GGML_OPENVINO_DEBUG_NODE` to materialise one node as an extra `ov::Result`. Both +of the subtlest fixes above were found by bisecting that way, comparing each +stage against a CPU-backend reference; neither logs anything when it goes wrong. + +## Known issues + +**π0 needs F32 on the GPU, and gets it by default.** The GPU plugin computes in +F16, which is most of why it is fast. π0 unrolls its whole 10-step denoise loop +inside a single graph, so that error compounds with nothing to reset it: its +continuous action dims land 4e-2 from an F32 reference and its gripper - a +saturating ±1 channel - crosses its threshold one step late. That reads as +max|delta| 1.7; on a robot it is a late grasp. `GGML_OPENVINO_GPU_PRECISION=f32` +puts it back at 6.5e-5, and `backend_init` defaults it for π0 alone because it +costs about 3x (383 ms -> 1,170 ms). Set `=f16` to override. No other arch needs +it. + +**Do not set `GGML_OPENVINO_CACHE_DIR`.** OpenVINO's on-disk blob cache reloads a +compiled graph that computes the wrong thing. A cold run against a fresh cache +directory is correct; the very next run, reading back the blobs it just wrote, is +not - max|delta| 1.2e-3 cold, 2.9e0 warm, with nothing logged, which for a policy +server is the worst possible failure mode. `backend_init` therefore clears the +variable and says so; `VLA_ALLOW_OV_CACHE=1` keeps it. Unverified guess at the +cause: the blob key does not capture something that differs between vla.cpp's +several graphs, so one graph gets another's blob - the same class of bug as the +in-process `graph_key` above, which is now keyed on shapes. In practice, pay the +compile once per process and leave it unset. + +**The NPU accepts three of the nine archs, and only two of those are correct.** +Check every NPU run against the startup banner: an unavailable NPU falls back to +the CPU plugin silently and would otherwise report excellent numbers that are not +NPU numbers at all. A partially-failing run still reports a wall-clock time, so +do not read a latency off a run whose actions did not come out. + +| Model | NPU outcome | +|---|---| +| SmolVLA | runs, 1.1e-2 (the compile config's dynamic quantization) | +| π0.5 | runs, 1.6e-3 | +| π0 | runs, but **1.7e0 wrong** - see below | +| VLA-JEPA | compiles and runs, returns all `NaN` (not diagnosed) | +| GR00T N1.5 / N1.6 / N1.7 | `NPUW: Assertion all_ok failed`, `partitioning.cpp:1350` | +| Evo-1 | compiler rejects: `Input channels '1025' is not aligned by '16'` | +| VLA-Adapter | compiler rejects: `Input channels '261' is not aligned by '16'` | + +The two alignment rejections are Intel's NPU compiler: 1025 is Evo-1's 1024 +patches plus a CLS token, 261 is VLA-Adapter's 256 plus 5. SmolVLA and π0.5 +happen to have 16-aligned sequence lengths. None of these are vla.cpp's doing. + +**π0 on the NPU is the same bug as π0 on the GPU, and here there is no remedy.** +Continuous dims 0-5 land at 4.0e-2 and the gripper flips at step 44, exactly as +on the GPU. But the GPU fix does not transfer: setting the inference precision to +F32 makes the NPU refuse to compile at all (`core.cpp:117`), because the hint +conflicts with the NPUW and dynamic-quantization config the NPU path sets up. So +`GGML_OPENVINO_GPU_PRECISION` is GPU-only by necessity, and π0 should not be run +on the NPU. + +**SmolVLA's `VLA_TIMING=phase` path is wrong under OpenVINO.** SmolVLA has a +second graph builder used when a caller asks for per-phase timings, and it does +not survive translation - max|delta| 1.9 on every device. The default +`TimingDetail::NONE` path, which is what `vla-server` and `vla-cli` use, is +correct, and on the native CPU backend the two paths agree exactly. Evo-1 and +π0.5 are unaffected on the same path, so this is specific to SmolVLA's second +graph; one hypothesis - that the split-graph guard sends it down the LLM path - +was tested and is wrong. Per-stage timings for SmolVLA are omitted from the +tables above. + +## What is left + +**Op coverage is no longer the blocker.** With RELU, GELU_ERF, NEG and SQR added +to the table, every ggml op the eleven in-tree archs build is translatable. The +one exception is `ggml_map_custom1`, used only by BitVLA, which pins its ggml +graph to the CPU backend by design - so an OpenVINO build leaves it there +regardless. That also makes `GGML_OP_SQR` **untested**: only BitVLA emits it. It +is in the table because it is a real gap in ggml-openvino, not because anything +here exercises it. + +**Untested archs.** OpenVLA-OFT is untested for want of a local checkpoint, not +because anything is known to block it; it uses an op set already covered by +VLA-Adapter. It is `~` in the README matrix; treat any untested arch that way +until it has actually produced actions. + +**Splitting across devices.** Intel's own +[π0.5 write-up](https://docs.openedgeplatform.intel.com/2026.1/OEP-articles/publications/optimizing-pi0.5-lva-model.html) +puts the vision encoder and language model on the iGPU and the action expert on +the NPU, with the KV cache as the only cross-device handoff. The toolchain does +not carry over - PyTorch exported to OpenVINO IR as three separate models, no +ggml - but the shape of the answer does: the two devices suit different stages, +and π0.5 on the NPU alone is already within 1.5x of the iGPU at a fraction of the +power. vla.cpp cannot make that split today because the core drives one backend +for a whole prediction. It would need a per-*stage* backend rather than a per-op +scheduler - the vision tower, the prefix and the action expert already hand off +through host memory, so the seam is in the right place - but that is an engine +change, not a backend one. diff --git a/examples/chat/README.md b/examples/chat/README.md index 29aaf4e..3ee40dc 100644 --- a/examples/chat/README.md +++ b/examples/chat/README.md @@ -46,7 +46,7 @@ Convert: SRC=/tmp/smolvlm2-500m-instruct OUT=~/data/$USER/smolvlm2-500m-instruct-gguf mkdir -p "$OUT" -CONV="PYTHONPATH=third_party/llama.cpp/gguf-py python3 third_party/llama.cpp/convert_hf_to_gguf.py" +CONV="PYTHONPATH=build/_deps/llama-src/gguf-py python3 build/_deps/llama-src/convert_hf_to_gguf.py" # text LM -> smolvlm2-500m-instruct-f16.gguf (819 MB) eval $CONV "$SRC" --outtype f16 \ @@ -70,7 +70,7 @@ cmake --build build-cuda --target llama-mtmd-cli ./build-cuda/bin/llama-mtmd-cli \ -m "$OUT/smolvlm2-500m-instruct-f16.gguf" \ --mmproj "$OUT/mmproj-smolvlm2-500m-instruct-f16.gguf" \ - --image third_party/llama.cpp/tools/mtmd/test-1.jpeg -p "Describe this image." --temp 0 + --image build/_deps/llama-src/tools/mtmd/test-1.jpeg -p "Describe this image." --temp 0 ``` > Shortcut: `HuggingFaceTB/SmolVLM2-500M-Video-Instruct` has **byte-identical @@ -123,7 +123,7 @@ python examples/chat/vlm_chat_client.py --addr tcp://localhost:5567 ```bash python examples/chat/vlm_chat_client.py --once \ - --image third_party/llama.cpp/tools/mtmd/test-1.jpeg \ + --image build/_deps/llama-src/tools/mtmd/test-1.jpeg \ -p "Describe this image in detail." -n 200 --temp 0 ``` diff --git a/scripts/install_ov.sh b/scripts/install_ov.sh new file mode 100644 index 0000000..28d450c --- /dev/null +++ b/scripts/install_ov.sh @@ -0,0 +1,391 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK_DIR="${SCRIPT_DIR}/.openvino_install_work" + +OS_ID="" +OS_VERSION="" + +log() { + printf '[install_openvino_runtime] %s\n' "$*" +} + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "Error: '$1' is required but not installed." >&2 + exit 1 + } +} + +# Digests of the two archives the defaults below pin. These land in /opt under +# sudo, so a bad download is a root-level problem. +OPENVINO_SHA256_2204="d701a115d3dc18088ff75b5b8e67a51fbf780022a3d40ee8ee7f2adfbd9915e6" +OPENVINO_SHA256_2404="6931e5a3c9b1fc9cb170137196df2c40489625703f2d184f511b7add2c110ef8" + +# verify_sha256 . An overridden version has no +# digest here, so fall back to the one the mirror publishes: that catches a +# truncated or corrupted download, not a compromised mirror. +verify_sha256() { + local path="$1" want="$2" url="$3" + if [[ -z "${want}" ]]; then + want="$(curl -fsSL "${url}.sha256" | awk 'NR==1 {print $1}')" + if [[ ! "${want}" =~ ^[0-9a-f]{64}$ ]]; then + echo "Error: no usable checksum published for ${url}" >&2 + exit 1 + fi + log "Version overridden, using the mirror's own checksum." + fi + if ! printf '%s %s\n' "${want}" "${path}" | sha256sum -c - >/dev/null; then + echo "Error: checksum mismatch for ${path}" >&2 + exit 1 + fi + log "Checksum OK: $(basename "${path}")" +} + +detect_os() { + if [[ ! -f /etc/os-release ]]; then + echo "Error: /etc/os-release not found; cannot detect Ubuntu version." >&2 + exit 1 + fi + + # shellcheck disable=SC1091 + source /etc/os-release + + OS_ID="${ID:-}" + OS_VERSION="${VERSION_ID:-}" + + if [[ "${OS_ID}" != "ubuntu" ]]; then + echo "Error: this installer supports Ubuntu only. Detected ID='${OS_ID:-unknown}'." >&2 + exit 1 + fi + + # Every package name below is amd64. Say so now, not after the first 404. + local arch + arch="$(uname -m)" + if [[ "${arch}" != "x86_64" ]]; then + echo "Error: Intel publishes these packages for x86_64 only. Detected '${arch}'." >&2 + exit 1 + fi +} + +prepare_common_tools() { + need_cmd bash + need_cmd sudo + need_cmd apt-get + need_cmd wget + need_cmd curl + need_cmd tar + need_cmd dpkg + need_cmd sha256sum + need_cmd find + need_cmd sort + + mkdir -p "${WORK_DIR}" +} + +prepare_common_dependencies() { + log "Installing common dependencies..." + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + libcurl4-openssl-dev \ + libtbb12 \ + cmake \ + ninja-build \ + python3-pip \ + curl \ + wget \ + tar \ + libopencl1 \ + ocl-icd-opencl-dev \ + opencl-headers \ + opencl-clhpp-headers \ + clinfo +} + +add_render_group() { + # USER is unbound, not empty, in a container or a cron shell, and set -u would + # abort here with the drivers already installed. + local target_user="${SUDO_USER:-${USER:-}}" + if [[ -z "${target_user}" ]]; then + return + fi + + if id -nG "${target_user}" | grep -qw render; then + log "User ${target_user} is already in the render group." + else + log "Adding ${target_user} to render group..." + sudo gpasswd -a "${target_user}" render || true + log "Re-login (or restart shell session) to apply render group membership." + fi +} + +install_gpu_2204() { + local download_dir="${WORK_DIR}/intel_gpu_2204" + local igc_base_url="https://github.com/intel/intel-graphics-compiler/releases/download/v2.10.8" + local crt_base_url="https://github.com/intel/compute-runtime/releases/download/25.13.33276.16" + local checksum_file="ww13.sum" + local packages=( + "${igc_base_url}/intel-igc-core-2_2.10.8+18926_amd64.deb" + "${igc_base_url}/intel-igc-opencl-2_2.10.8+18926_amd64.deb" + "${crt_base_url}/intel-level-zero-gpu-dbgsym_1.6.33276.16_amd64.ddeb" + "${crt_base_url}/intel-level-zero-gpu_1.6.33276.16_amd64.deb" + "${crt_base_url}/intel-opencl-icd-dbgsym_25.13.33276.16_amd64.ddeb" + "${crt_base_url}/intel-opencl-icd_25.13.33276.16_amd64.deb" + "${crt_base_url}/libigdgmm12_22.7.0_amd64.deb" + ) + + log "Installing Intel GPU drivers for Ubuntu 22.04..." + mkdir -p "${download_dir}" + cd "${download_dir}" + + for url in "${packages[@]}"; do + wget --no-continue "${url}" + done + wget --no-continue "${crt_base_url}/${checksum_file}" + + sha256sum --ignore-missing -c "${checksum_file}" + + shopt -s nullglob + local artifacts=( *.deb *.ddeb ) + shopt -u nullglob + if [[ ${#artifacts[@]} -eq 0 ]]; then + echo "Error: no GPU package files found for Ubuntu 22.04." >&2 + exit 1 + fi + + sudo dpkg -i "${artifacts[@]}" || sudo apt-get install -f -y + cd "${SCRIPT_DIR}" + rm -rf "${download_dir}" +} + +install_npu_2204() { + local download_dir="${WORK_DIR}/intel_npu_2204" + local npu_tarball="linux-npu-driver-v1.26.0.20251125-19665715237-ubuntu2204.tar.gz" + local npu_url="https://github.com/intel/linux-npu-driver/releases/download/v1.26.0/${npu_tarball}" + local level_zero_deb="level-zero_1.24.2+u22.04_amd64.deb" + local level_zero_url="https://github.com/oneapi-src/level-zero/releases/download/v1.24.2/${level_zero_deb}" + + log "Installing Intel NPU drivers for Ubuntu 22.04..." + mkdir -p "${download_dir}" + cd "${download_dir}" + + # Download before purging. The other order leaves a machine with no NPU driver + # at all if the fetch fails. + wget --no-continue "${npu_url}" + wget --no-continue "${level_zero_url}" + tar -xf "${npu_tarball}" + + mapfile -t npu_debs < <(find . -type f -name '*.deb' ! -name 'level-zero*.deb' | sort) + if [[ ${#npu_debs[@]} -eq 0 ]]; then + echo "Error: no Intel NPU .deb packages found for Ubuntu 22.04." >&2 + exit 1 + fi + + sudo dpkg --purge --force-remove-reinstreq \ + intel-driver-compiler-npu \ + intel-fw-npu \ + intel-level-zero-npu \ + intel-level-zero-npu-dbgsym || true + + sudo dpkg -i "${npu_debs[@]}" || sudo apt-get install -f -y + sudo dpkg -i "${level_zero_deb}" || sudo apt-get install -f -y + + add_render_group + + cd "${SCRIPT_DIR}" + rm -rf "${download_dir}" +} + +install_runtime_2204() { + local download_dir="${WORK_DIR}/openvino_runtime_2204" + local openvino_version="${OPENVINO_VERSION:-2025.3}" + local openvino_build="${OPENVINO_BUILD:-19807.44526285f24}" + local openvino_archive="openvino_toolkit_ubuntu22_${openvino_version}.0.${openvino_build}_x86_64.tgz" + local openvino_dirname="openvino_toolkit_ubuntu22_${openvino_version}.0.${openvino_build}_x86_64" + local openvino_url="https://storage.openvinotoolkit.org/repositories/openvino/packages/${openvino_version}/linux/${openvino_archive}" + local install_root="${INSTALL_ROOT:-/opt/intel}" + local install_dir="${install_root}/openvino_${openvino_version}" + local symlink_path="${install_root}/openvino" + local archive_path="${download_dir}/openvino_${openvino_version}.tgz" + local expected_sha="" + if [[ "${openvino_version}" == "2025.3" && "${openvino_build}" == "19807.44526285f24" ]]; then + expected_sha="${OPENVINO_SHA256_2204}" + fi + + log "Installing OpenVINO runtime for Ubuntu 22.04..." + sudo mkdir -p "${install_root}" + mkdir -p "${download_dir}" + + curl -fL "${openvino_url}" --output "${archive_path}" + verify_sha256 "${archive_path}" "${expected_sha}" "${openvino_url}" + rm -rf "${download_dir:?}/${openvino_dirname}" + tar -xf "${archive_path}" -C "${download_dir}" + + sudo rm -rf "${install_dir}" + sudo mv "${download_dir}/${openvino_dirname}" "${install_dir}" + sudo ln -sfn "openvino_${openvino_version}" "${symlink_path}" + + if [[ ! -f "${symlink_path}/setupvars.sh" ]]; then + echo "Error: ${symlink_path}/setupvars.sh was not found after installation." >&2 + exit 1 + fi + + rm -rf "${download_dir}" +} + +install_gpu_2404() { + local download_dir="${WORK_DIR}/intel_gpu_2404" + local igc_base_url="https://github.com/intel/intel-graphics-compiler/releases/download/v2.36.3" + local crt_base_url="https://github.com/intel/compute-runtime/releases/download/26.22.38646.4" + local checksum_file="ww22.sum" + local packages=( + "${igc_base_url}/intel-igc-core-2_2.36.3+21719_amd64.deb" + "${igc_base_url}/intel-igc-opencl-2_2.36.3+21719_amd64.deb" + "${crt_base_url}/intel-ocloc-dbgsym_26.22.38646.4-0_amd64.ddeb" + "${crt_base_url}/intel-ocloc_26.22.38646.4-0_amd64.deb" + "${crt_base_url}/intel-opencl-icd-dbgsym_26.22.38646.4-0_amd64.ddeb" + "${crt_base_url}/intel-opencl-icd_26.22.38646.4-0_amd64.deb" + "${crt_base_url}/libigdgmm12_22.10.0_amd64.deb" + "${crt_base_url}/libze-intel-gpu1-dbgsym_26.22.38646.4-0_amd64.ddeb" + "${crt_base_url}/libze-intel-gpu1_26.22.38646.4-0_amd64.deb" + ) + + log "Installing Intel GPU drivers for Ubuntu 24.04..." + mkdir -p "${download_dir}" + cd "${download_dir}" + + for url in "${packages[@]}"; do + wget --no-continue "${url}" + done + wget --no-continue "${crt_base_url}/${checksum_file}" + + sha256sum --ignore-missing -c "${checksum_file}" + + shopt -s nullglob + local artifacts=( *.deb *.ddeb ) + shopt -u nullglob + if [[ ${#artifacts[@]} -eq 0 ]]; then + echo "Error: no GPU package files found for Ubuntu 24.04." >&2 + exit 1 + fi + + sudo dpkg -i "${artifacts[@]}" || sudo apt-get install -f -y + cd "${SCRIPT_DIR}" + rm -rf "${download_dir}" +} + +install_npu_2404() { + local download_dir="${WORK_DIR}/intel_npu_2404" + local npu_release="v1.33.0" + local npu_archive="linux-npu-driver-v1.33.0.20260529-26625960453-ubuntu2404.tar.gz" + local npu_url="https://github.com/intel/linux-npu-driver/releases/download/${npu_release}/${npu_archive}" + local npu_packages=( + intel-driver-compiler-npu + intel-fw-npu + intel-level-zero-npu + intel-level-zero-npu-dbgsym + ) + + log "Installing Intel NPU drivers for Ubuntu 24.04..." + mkdir -p "${download_dir}" + cd "${download_dir}" + + # Download before purging, so a failed fetch does not leave the machine with + # no NPU driver at all. + wget --no-continue "${npu_url}" + tar -xf "${npu_archive}" + + shopt -s nullglob + local debs=( *.deb ) + shopt -u nullglob + if [[ ${#debs[@]} -eq 0 ]]; then + echo "Error: no Intel NPU .deb packages found for Ubuntu 24.04." >&2 + exit 1 + fi + + sudo dpkg --purge --force-remove-reinstreq "${npu_packages[@]}" || true + sudo dpkg -i "${debs[@]}" || sudo apt-get install -f -y + + # The Level Zero loader the NPU plugin dlopens. 22.04 needs it from GitHub; + # 24.04 has it in the archive. See docs/backend/ov.md for ZE_ENABLE_ALT_DRIVERS. + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libze1 + + add_render_group + + cd "${SCRIPT_DIR}" + rm -rf "${download_dir}" +} + +install_runtime_2404() { + local download_dir="${WORK_DIR}/openvino_runtime_2404" + local openvino_version="${OPENVINO_VERSION:-2026.2.1}" + local openvino_build="${OPENVINO_BUILD:-21919.ede283a88e3}" + local openvino_archive="openvino_toolkit_ubuntu24_${openvino_version}.${openvino_build}_x86_64.tgz" + local openvino_dirname="openvino_toolkit_ubuntu24_${openvino_version}.${openvino_build}_x86_64" + local openvino_url="https://storage.openvinotoolkit.org/repositories/openvino/packages/${openvino_version}/linux/${openvino_archive}" + local install_root="${INSTALL_ROOT:-/opt/intel}" + local install_dir="${install_root}/openvino_${openvino_version}" + local symlink_path="${install_root}/openvino" + local archive_path="${download_dir}/openvino_${openvino_version}.tgz" + local expected_sha="" + if [[ "${openvino_version}" == "2026.2.1" && "${openvino_build}" == "21919.ede283a88e3" ]]; then + expected_sha="${OPENVINO_SHA256_2404}" + fi + + log "Installing OpenVINO runtime for Ubuntu 24.04..." + sudo mkdir -p "${install_root}" + mkdir -p "${download_dir}" + + curl -fL "${openvino_url}" --output "${archive_path}" + verify_sha256 "${archive_path}" "${expected_sha}" "${openvino_url}" + rm -rf "${download_dir:?}/${openvino_dirname}" + tar -xf "${archive_path}" -C "${download_dir}" + + sudo rm -rf "${install_dir}" + sudo mv "${download_dir}/${openvino_dirname}" "${install_dir}" + sudo ln -sfn "openvino_${openvino_version}" "${symlink_path}" + + if [[ ! -f "${symlink_path}/setupvars.sh" ]]; then + echo "Error: ${symlink_path}/setupvars.sh was not found after installation." >&2 + exit 1 + fi + + rm -rf "${download_dir}" +} + +run_installation() { + case "${OS_VERSION}" in + 22.04) + install_gpu_2204 + install_npu_2204 + install_runtime_2204 + ;; + 24.04) + install_gpu_2404 + install_npu_2404 + install_runtime_2404 + ;; + *) + echo "Error: unsupported Ubuntu version '${OS_VERSION:-unknown}'. Supported versions: 22.04, 24.04." >&2 + exit 1 + ;; + esac +} + +main() { + detect_os + prepare_common_tools + prepare_common_dependencies + + log "Detected Ubuntu ${OS_VERSION}." + run_installation + rm -rf "${WORK_DIR}" + + log "All OpenVINO installation steps completed successfully." + log "To load OpenVINO in current shell: source /opt/intel/openvino/setupvars.sh" +} + +main "$@" diff --git a/scripts/llama_tag.sh b/scripts/llama_tag.sh new file mode 100755 index 0000000..53afadb --- /dev/null +++ b/scripts/llama_tag.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Print the llama.cpp tag CMakeLists.txt pins. Two consumers read it - the CI +# cache key and print_versions.sh - and both used to grep for it themselves. +# That grep broke silently once when the tag moved behind ${VLA_LLAMA_TAG} and +# loudly again when the default moved behind another variable, so it lives here. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CMAKE="${1:-${ROOT}/CMakeLists.txt}" + +tag="$(grep -m1 -oE 'set\(_vla_llama_tag_default "[^"]+"' "${CMAKE}" | grep -oE '"[^"]+"' | tr -d '"' || true)" + +if [[ -z "${tag}" ]]; then + echo "llama_tag: no _vla_llama_tag_default in ${CMAKE}" >&2 + exit 1 +fi +echo "${tag}" diff --git a/scripts/patch_ggml_openvino.py b/scripts/patch_ggml_openvino.py new file mode 100755 index 0000000..0bbffba --- /dev/null +++ b/scripts/patch_ggml_openvino.py @@ -0,0 +1,881 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Thirteen fixes to the fetched ggml OpenVINO backend. + +ggml-openvino is written against llama.cpp's graphs: one decoder-only +transformer, one position input, an F16 KV cache. vla.cpp drives it with vision +towers and action experts instead, which is legal ggml but nothing the backend +has seen. Eight of the hunks below narrow an llama.cpp-shaped assumption back to +the ggml contract, one fills a gap in the op table, and two are about the path +those graphs take. Together they are what lets SmolVLA and pi0.5 run end to end +on the CPU, GPU and NPU plugins. Numbers 4 and 5 are the ones worth upstreaming. + +A hunk that applies is not a hunk that runs. Whenever VLA_LLAMA_TAG moves, check +that each patched function still has a live caller, not just that its anchor +still matches: b10729 moved the position-input naming out of the GgmlOvDecoder +member that fix 4 patches and into a free function, and the fix went silently +dead while applying cleanly. + +See docs/backend/ov.md for the measured results and for what is still blocked. + + 1. ggml-openvino-extra.cpp - pick the *Intel* OpenCL platform. + `GGML_OPENVINO_DEVICE=GPU` builds an OpenVINO remote context on an OpenCL + queue and takes the first platform the ICD loader reports. With more than + one runtime installed (an NVIDIA card next to the Intel iGPU, POCL, + Rusticl) that is whichever `/etc/OpenCL/vendors/*.icd` sorted first, and + the GPU plugin only accepts an Intel context -- it aborts at startup with + "Incompatible OpenCL runtime: program is not in expected ELF format". + Selecting by `CL_PLATFORM_VENDOR` leaves single-runtime boxes unchanged. + + 2. ggml-decoder.cpp - narrow RESHAPE op_case 3. + Case 3 is the KV-cache flatten, `[512,1024,1,1] -> [1,524288,1,1]`, and it + emits a shape with `-1` in dim 2 and 1 in dim 3. Its guard only tests + `src->ne[0]*ne[1]*ne[2] == node->ne[1]`, which also matches the kernel + reshape inside `ggml_conv_2d` (`[16,16,3,768] -> [768,768]`) and mangles + it. The real case always has `node->ne[0] == 1`; requiring that sends the + conv kernel to case 6, the plain reshape. + + 3. openvino/op/flash_attn_ext.cpp - convert K/V to F16 with Q. + The translator converts Q, the mask and the scale to F16 because + llama.cpp's KV cache already is. vla.cpp keeps K/V in F32, and OpenVINO's + SDPA rejects mixed input types ("Mixed input types are not supported"). + Converting K/V too matches the precision the translator already chose. + + 4. ggml-decoder.{h,cpp} - stop distinct position inputs aliasing each other. + Every tensor feeding a ROPE's second input is renamed to one graph + parameter called "inp_pos", because an llama.cpp graph has exactly one. + add_rope_sin_cos() then builds a single shared sin/cos table from it. Every + vla.cpp arch has several position tensors -- SmolVLA passes a prefill, a + full and a rebased one -- so they alias each other and every ROPE takes the + table built from whichever won, which fails shape inference: + "Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32]) + Argument shapes are inconsistent." + When the graph has more than one, keep each tensor's own name. Nothing is + then called "inp_pos", so translate_rope() falls back to building sin/cos + per op from its own position input. Single-position graphs are untouched. + This one is what carries an arch through to a full prediction. + + Patch the free function get_tensor_graph_input_ov_name() in + ggml-decoder.cpp, not just the GgmlOvDecoder member. b10729 relocated the + naming into that free function and left the member behind with no callers; + patching only the member applies cleanly and does nothing at all. Both are + patched here so the fix survives whichever one upstream keeps. + + 5. utils.{h,cpp} - cache what the naive path compiles. + The dynamic and static paths keep a `graph_key`-indexed cache of the + decoder and the compiled infer request; the naive path has none, so it + rebuilt the decoder, re-converted the model and called compile_model() on + every single ggml_backend_graph_compute. That is the dominant cost once a + real graph goes through it: SmolVLA on the CPU plugin drops from 22.7 s to + 1.4 s per prediction with the cache in place. A hit rebinds the cached + decoder to the new graph through the existing update_io(), which is how the + dynamic path already handles freshly built tensors. + The key is `naive_key`, not `graph_key`: the latter is n_nodes plus the + first and last node name, which two graphs of the same size can share, and + a compiled model is bound to the shapes it was built for. Reusing one + across a shape change returns another graph's answer with no error, so the + key mixes in every node's op and shape. The map is bounded; see the comment + on the flush. + + 6. openvino/op_table.cpp - get both GELU flavours right. + ggml has two: GGML_UNARY_OP_GELU is the tanh approximation, GGML_UNARY_OP_ + GELU_ERF is the exact one. The table mapped GELU onto ov's Gelu, which + defaults to erf, and had no entry for GELU_ERF at all. So the tanh op was + computed as erf, and an arch using the erf op could not run at all - and + with no per-op CPU fallback in the core, "could not run" means the whole + prediction. Small per node, but a vision tower has dozens and the error + compounds: setting the mode explicitly moved VLA-JEPA from 5.5e-3 to 1.1e-4 + and GR00T N1.5 from 2.7e-2 to 6.0e-4, turning both from "runs but drifts" + into supported. The highest-yield single fix here. + RELU, NEG and SQR were missing here too; upstream added all three in b10729. + + 7. openvino/utils.cpp - give a folded weight its full rank before slicing. + A ggml tensor that is 2-D folds in as a rank-2 ov constant, which is what a + GEMM operand wants, but process_view_input_new() indexes a viewed tensor at + its full ggml rank, so the slice axis lands outside it: + "Slice (Constant aex.blk.0.attn_in.weight[0]:bf16[2688,896], ...) + Axis 2 out of the tensor rank range [-2, 1]." + Evo-1 hits it by viewing Q, K and V out of one fused attn_in weight. + Left-pad the input with 1s, which is the shape ggml gave it anyway. + + 8. openvino/op/concat.cpp - align input ranks. + The same rank-2 constants reach CONCAT, which unlike the broadcasting + elementwise ops needs both inputs at the graph's rank or the axis falls + outside them. Evo-1 concatenates a CLS weight onto 4-D patch embeddings; + SmolVLA concatenates a precomputed time tile onto a 4-D activation. Padding + here is what saves each arch from working around it, e.g. by moving + SmolVLA's time tiles into a buffer of their own. + + 9. openvino/utils.cpp - bound the interleaved-mrope sector cycle by sections. + ggml's IMROPE cycles t/h/w by sector % 3, but only while the sector is + inside 3 * sections[k]; past that it falls through to the fourth position + stream (ggml_rope_cache_init in ggml/src/ggml-cpu/ops.cpp). The translator + cycled unconditionally, so with sections {24,20,20,0} and n_dims 128 + sectors 61 and 62 took h and w instead of e. NOTE: this matches the ggml + reference but had no measurable effect on any arch tested here, because the + fourth stream happens to carry the same positions as the first. Kept + because it removes a real divergence from the reference, not because a + measurement demanded it. + + 10. utils.cpp - make the naive-path graph-size threshold settable. + Graphs under 20 nodes bypass the LLM decoder and translate literally, with + static shapes and no KV-cache inference. That literal path is the one that + suits a vision tower, but a vision tower is ~450 nodes. The constant + becomes `GGML_OPENVINO_NAIVE_GRAPH_SIZE`; src/backend.h defaults it high + for vla.cpp and an explicit setting still wins. Parsed with strtol, because + atoi turns junk into 0 and that would send every graph down the LLM builder + with nothing said. + + 11. ggml-decoder.cpp - require a ROPE before taking PERMUTE op_case 2. + op_case 2 rewrites the tensor as [n_seq, -1, n_heads, head_size] and only + then transposes, which is correct for llama.cpp's rope'd query and nothing + else. The classifier reached it for ANY permute whose source is a view of a + non-leaf, so GR00T N1.7's DiT cross-attention V - ggml_permute(view, 1,2,0,3) + over a fused KV projection - was reshaped into a shape unrelated to it and + came out with its elements rearranged: V was 139% wrong while K, which uses + permute(0,2,1,3) and happened to survive the same rewrite, was 0.04%. That + turned into a 27% error in the final actions. Walking the view/reshape/cont + chain and requiring a ROPE at the end sends every other permute to op_case 1, + the plain transpose. This is what makes GR00T N1.7 correct (27% -> 0.005%). + + 12. ggml-decoder.cpp + openvino/op/add.cpp - do not stack two elementwise adds + on a GEMM. The GPU plugin folds elementwise ops into the preceding GEMM as + post-ops. Given ADD(ADD(residual, GEMM), graph_input) it folds both, and the + second operand is silently lost: the result equals the inner add, as though + the outer one never ran. Nothing is logged. An llama.cpp graph never builds + that chain - one residual add per sub-block - but a VLA does, wherever a + tower's features are added on top of an FFN residual. It cost VLA-JEPA, + GR00T N1.7 and pi0 their GPU support, while every other arch was unaffected. + Addition is associative, so re-hang the outer add on the inner one's + non-GEMM operand: the GEMM keeps exactly one post-op and the arithmetic is + unchanged. op_case 2/3 records which of the inner add's operands is the + GEMM, because ggml's operand order is not fixed. + Found by bisecting VLA-JEPA with GGML_OPENVINO_DEBUG_NODE: its ViT and DiT + graphs matched the CPU plugin to 0.2%, the VLM prefill was already wrong at + the end of layer 0, and within that layer the divergence was one node - the + FFN residual add under the deepstack add. The same fusion path already has a + known defect with broadcast DIV; see ggml_backend_openvino_supports_op. + + 13. ggml-openvino-extra.cpp - expose the GPU plugin's inference precision. + The GPU plugin computes in f16 unless told otherwise, which is most of why + it is fast, and for nearly every graph that is the right trade. It is not + the right trade for a graph carrying a long serial chain - a flow-matching + denoise loop unrolled inside one graph - because the error compounds across + every step with nothing to reset it, and a saturating output channel can + then cross its threshold in the wrong place. pi0 does exactly that: on the + GPU its continuous action dims land 4e-2 from an F32 reference and its + bistable gripper flips one step early, which is a 1.7 max|delta| on a metric + and a late grasp on a robot. GGML_OPENVINO_GPU_PRECISION=f32 puts it at + 6.5e-5. Exposed rather than forced: f32 costs about 3x on this plugin, so + src/backend.h defaults it for pi0 alone and an explicit setting still wins. + +Idempotent - re-running on a patched tree is a no-op, so a reconfigure that +re-populates the FetchContent source dir is safe either way. + +Usage: scripts/patch_ggml_openvino.py [] +""" + +import pathlib +import re +import sys + +HELPER = """// vla.cpp: select the Intel OpenCL platform. With several OpenCL runtimes +// installed the first platform is not always Intel's, and the OpenVINO GPU +// plugin only accepts an Intel context. Cached: the ICD list cannot change +// under a running process. +static cl_platform_id ggml_openvino_get_intel_platform() { + static cl_platform_id platform = nullptr; + static bool searched = false; + if (searched) { + return platform; + } + searched = true; + + cl_uint n_platforms = 0; + if (clGetPlatformIDs(0, nullptr, &n_platforms) != CL_SUCCESS || n_platforms == 0) { + return nullptr; + } + std::vector platforms(n_platforms); + if (clGetPlatformIDs(n_platforms, platforms.data(), nullptr) != CL_SUCCESS) { + return nullptr; + } + + for (cl_platform_id p : platforms) { + char vendor[256] = ""; + if (clGetPlatformInfo(p, CL_PLATFORM_VENDOR, sizeof(vendor), vendor, nullptr) != CL_SUCCESS) { + continue; + } + if (strstr(vendor, "Intel") != nullptr) { + platform = p; + break; + } + } + return platform; +} + +""" + +USM_LOOKUP = """ cl_platform_id platform; + if (clGetPlatformIDs(1, &platform, nullptr) == CL_SUCCESS) { + fn = (%s_fn) clGetExtensionFunctionAddressForPlatform(platform, "%s"); +""" + +USM_LOOKUP_NEW = """ cl_platform_id platform = ggml_openvino_get_intel_platform(); + if (platform != nullptr) { + fn = (%s_fn) clGetExtensionFunctionAddressForPlatform(platform, "%s"); +""" + +NAIVE_COMPUTE_OLD = """enum ggml_status naive_compute(ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config) { + if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { + return GGML_STATUS_SUCCESS; + } + + bool naive = true; + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); + auto decoder = std::make_shared(cgraph, model_weights); + auto input_model = std::make_shared(decoder); + auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + ov::serialize(model, "IR_naive.xml"); + } + + std::shared_ptr infer_request; + auto remote_context = ggml_openvino_get_remote_context(); + if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { + // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved + core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::PERFORMANCE)); + } else { + core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); + } + if (remote_context.has_value()) { + infer_request = std::make_shared( + core.compile_model(model, remote_context.value(), config).create_infer_request()); + } else { + infer_request = + std::make_shared(core.compile_model(model, device, config).create_infer_request()); + } + + auto ov_params = model->get_parameters();""" + +NAIVE_COMPUTE_NEW = """enum ggml_status naive_compute(ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config, + std::shared_ptr r_ctx) { + if (cgraph->n_nodes == 1 && (cgraph->nodes[0]->op == GGML_OP_NONE || cgraph->nodes[0]->op == GGML_OP_VIEW)) { + return GGML_STATUS_SUCCESS; + } + + // vla.cpp: reuse the decoder, the converted model and the compiled infer + // request across calls on the same graph, the way the dynamic and static + // paths already do. Conversion plus compile_model dominates a naive call, so + // without this every graph_compute pays it again. + static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + const naive_key key(cgraph); + + std::shared_ptr entry; + bool cache_hit = false; + if (cache_enabled && r_ctx != nullptr) { + std::lock_guard lock(r_ctx->ctx_mutex); + auto it = r_ctx->naive_cache.find(key); + if (it != r_ctx->naive_cache.end()) { + entry = it->second; + cache_hit = true; + } else { + // Each entry holds a compiled model, so this cannot grow forever. + // Flush rather than evict: a caller sees a handful of shapes, and an + // LRU is only worth its bookkeeping once that stops being true. + if (r_ctx->naive_cache.size() >= 32) { + r_ctx->naive_cache.clear(); + } + entry = std::make_shared(); + r_ctx->naive_cache[key] = entry; + } + } else { + entry = std::make_shared(); + } + + // One graph at a time: an ov::InferRequest is not re-entrant, and a hit + // rebinds the decoder to this cgraph. + std::lock_guard entry_lock(entry->mutex); + + bool naive = true; + std::shared_ptr decoder; + std::shared_ptr model; + std::shared_ptr infer_request; + + if (cache_hit && entry->infer_request != nullptr) { + decoder = entry->decoder; + model = entry->model; + infer_request = entry->infer_request; + // Same shapes, new tensors: point the decoder at this call's graph. + decoder->update_io(cgraph); + } else { + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); + decoder = std::make_shared(cgraph, model_weights); + auto input_model = std::make_shared(decoder); + model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { + ov::serialize(model, "IR_naive.xml"); + } + + auto remote_context = ggml_openvino_get_remote_context(); + if (cgraph->nodes[0]->op == GGML_OP_MUL_MAT) { + // TODO ACCURACY hint triggers a bug in GPU plugin/driver on Lunar Lake. Remove once CVS-182166 is resolved + core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::PERFORMANCE)); + } else { + core.set_property(device, ov::hint::execution_mode(ov::hint::ExecutionMode::ACCURACY)); + } + if (remote_context.has_value()) { + infer_request = std::make_shared( + core.compile_model(model, remote_context.value(), config).create_infer_request()); + } else { + infer_request = + std::make_shared(core.compile_model(model, device, config).create_infer_request()); + } + + entry->decoder = decoder; + entry->model = model; + entry->infer_request = infer_request; + } + + auto ov_params = model->get_parameters();""" + +# file -> [(anchor, replacement), ...]. Every anchor must match exactly once. +EDITS = { + "ggml/src/ggml-openvino/ggml-openvino-extra.cpp": [ + ("#include \n", "#include \n#include \n"), + ("void ggml_openvino_device_config::init() {", HELPER + "void ggml_openvino_device_config::init() {"), + ( + """ cl_int err; + cl_platform_id platform; + err = clGetPlatformIDs(1, &platform, nullptr); + if (err != CL_SUCCESS) { + GGML_LOG_ERROR("Failed to get OpenCL platform: %d\\n", err); + return; + } +""", + """ cl_int err; + cl_platform_id platform = ggml_openvino_get_intel_platform(); + if (platform == nullptr) { + GGML_LOG_ERROR("Failed to find an Intel OpenCL platform\\n"); + return; + } +""", + ), + (USM_LOOKUP % (("clEnqueueMemFillINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemFillINTEL",) * 2)), + (USM_LOOKUP % (("clEnqueueMemcpyINTEL",) * 2), USM_LOOKUP_NEW % (("clEnqueueMemcpyINTEL",) * 2)), + ( + """ "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + };""", + """ "GGML_OPENVINO_LOG_UNSUPPORTED_OPS", + // vla.cpp: f16 (default) or f32 for the GPU plugin's inference precision. + "GGML_OPENVINO_GPU_PRECISION", + };""", + ), + ( + """ } else if (cache_dir && strlen(cache_dir) > 0) { + compile_config.insert(ov::cache_dir(cache_dir)); + compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE)); + }""", + """ } else if (cache_dir && strlen(cache_dir) > 0) { + compile_config.insert(ov::cache_dir(cache_dir)); + compile_config.insert(ov::cache_mode(ov::CacheMode::OPTIMIZE_SIZE)); + } + + // vla.cpp: the GPU plugin computes in f16 unless told otherwise, which is why + // it is fast. A model whose graph carries a long serial chain -- a denoise + // loop unrolled inside one graph -- compounds that across every step, and a + // saturating output channel can then cross its threshold in the wrong place. + // Exposed rather than forced: f32 costs roughly 3x on this plugin. + if (device_name == "GPU") { + const char * gpu_prec = ggml_openvino_getenv_str("GGML_OPENVINO_GPU_PRECISION", "f16"); + if (strcmp(gpu_prec, "f32") == 0) { + compile_config.insert(ov::hint::inference_precision(ov::element::f32)); + } else if (strcmp(gpu_prec, "f16") != 0) { + GGML_LOG_WARN("GGML OpenVINO Backend: GGML_OPENVINO_GPU_PRECISION=%s is not f16 or f32, ignoring\\n", + gpu_prec); + } + }""", + ), + ], + "ggml/src/ggml-openvino/openvino/op/add.cpp": [ + ( + """ auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + auto res = std::make_shared(input_0, input_1);""", + """ auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + + // vla.cpp: re-hang the outer add on the inner one's non-GEMM operand so the + // GEMM is left with a single post-op. Addition is associative. + const int oc = context.get_op_case(); + if (oc == 2 || oc == 3) { + auto inner = input_0.get_node_shared_ptr(); + if (inner->get_input_size() == 2) { + const size_t keep = (oc == 2) ? 0 : 1; + const size_t fold = 1 - keep; + auto folded = std::make_shared(inner->input_value(fold), input_1); + auto res2 = std::make_shared(inner->input_value(keep), folded); + return rename_outputs_with_suffix({res2}, context.get_name()); + } + } + + auto res = std::make_shared(input_0, input_1);""", + ), + ], + "ggml/src/ggml-openvino/openvino/op/concat.cpp": [ + ( + """#include +#include """, + """#include +#include +#include +#include """, + ), + ("#include ", "#include \n#include "), + ( + """ const auto axis = static_cast(rank - 1 - ggml_dim); + auto res = std::make_shared(OutputVector{input_0, input_1}, axis);""", + """ // vla.cpp: a weight that is 2-D in ggml is folded in as a rank-2 constant, + // because that is what a GEMM operand wants. Concat is the one op where that + // matters: it needs both inputs at the graph's rank, or the axis computed + // below falls outside them. Left-pad the shorter one with 1s, which is the + // shape ggml gave it anyway. + auto align_rank = [rank](ov::Output in) { + const auto & ps = in.get_partial_shape(); + if (ps.rank().is_dynamic() || ps.rank().get_length() >= rank) { + return in; + } + std::vector axes(rank - ps.rank().get_length()); + std::iota(axes.begin(), axes.end(), 0); + auto axes_node = ov::op::v0::Constant::create(ov::element::i64, {axes.size()}, axes); + return ov::Output(std::make_shared(in, axes_node)); + }; + input_0 = align_rank(input_0); + input_1 = align_rank(input_1); + + const auto axis = static_cast(rank - 1 - ggml_dim); + auto res = std::make_shared(OutputVector{input_0, input_1}, axis);""", + ), + ], + "ggml/src/ggml-openvino/openvino/utils.cpp": [ + ("#include ", "#include \n#include "), + ( + """ std::vector gather_indices(n_dims_half); + for (size_t j = 0; j < n_dims_half; j++) { + gather_indices[j] = j % 3; + factor[j] = std::pow(theta_scale, j); + }""", + """ // vla.cpp: ggml's interleaved mrope cycles t/h/w by sector % 3, but only + // while the sector is still inside 3 * sections[k]; past that it falls + // through to the fourth position stream. Ignoring the bound sends the + // tail sectors to the wrong stream -- with sections {24,20,20,0} and + // n_dims 128, sectors 61 and 62 take h and w instead of e. See + // ggml_rope_cache_init in ggml/src/ggml-cpu/ops.cpp. + const int32_t * sections = rope_params + 11; + std::vector gather_indices(n_dims_half); + for (size_t j = 0; j < n_dims_half; j++) { + const int sector = (int) j; + int64_t stream = 3; + if (sector % 3 == 1 && sector < 3 * sections[1]) { + stream = 1; + } else if (sector % 3 == 2 && sector < 3 * sections[2]) { + stream = 2; + } else if (sector % 3 == 0 && sector < 3 * sections[0]) { + stream = 0; + } + gather_indices[j] = stream; + factor[j] = std::pow(theta_scale, j); + }""", + ), + ( + "#include ", + "#include \n#include ", + ), + ( + """ size_t view_input_size = context.get_view_input_size(input_index); + if (view_input_size == 0) { + // No view inputs, return the input as is + return input; + } +""", + """ size_t view_input_size = context.get_view_input_size(input_index); + if (view_input_size == 0) { + // No view inputs, return the input as is + return input; + } + + // vla.cpp: a ggml tensor that is 2-D folds in as a rank-2 ov constant, which + // is what a GEMM operand wants, but every slice below indexes the tensor at + // its full ggml rank. Left-pad with 1s so the axes line up. Evo-1 hits this + // by viewing Q/K/V out of one fused attn_in weight. + { + const auto src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, 0); + const auto & in_ps = input.get_partial_shape(); + if (in_ps.rank().is_static() && (size_t) in_ps.rank().get_length() < src_ggml_shape.size()) { + std::vector axes(src_ggml_shape.size() - (size_t) in_ps.rank().get_length()); + std::iota(axes.begin(), axes.end(), 0); + input = std::make_shared( + input, ov::op::v0::Constant::create(ov::element::i64, {axes.size()}, axes)); + } + } +""", + ), + ], + "ggml/src/ggml-openvino/openvino/op_table.cpp": [ + ( + """namespace ov { +namespace frontend { +namespace ggml { + +std::unordered_map get_supported_ops() {""", + """namespace ov { +namespace frontend { +namespace ggml { + +namespace op { +// vla.cpp: ggml's GGML_UNARY_OP_GELU is the *tanh* approximation (its CPU kernel +// additionally reads an fp16 lookup table); ov::op::v7::Gelu defaults to the +// exact erf formulation. Mapping the tanh op onto erf is a real approximation +// mismatch -- small per node, but a vision tower has dozens of them and the +// error compounds through the whole encoder. +static OutputVector translate_gelu_tanh(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input, ov::op::GeluApproximationMode::TANH); + return rename_outputs_with_suffix({res}, context.get_name()); +} +} // namespace op + +std::unordered_map get_supported_ops() {""", + ), + ( + """ {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input },""", + """ {"GGML_UNARY_OP_GELU", op::translate_gelu_tanh }, + // vla.cpp: tanh approximation for GELU, exact erf for GELU_ERF. ov's Gelu + // defaults to erf, so the tanh variant must set its mode explicitly. + {"GGML_UNARY_OP_GELU_ERF", op::translate_1to1_match_1_input },""", + ), + ], + "ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp": [ + ( + """ auto q = std::make_shared(q_f32, ov::element::f16);""", + """ auto q = std::make_shared(q_f32, ov::element::f16); + // vla.cpp: Q, the mask and the scale below are all forced to F16 because + // llama.cpp's KV cache already is. K/V that arrive as F32 have to come along + // or SDPA rejects the mix. + if (k.get_element_type() != ov::element::f16) { + k = std::make_shared(k, ov::element::f16); + } + if (v.get_element_type() != ov::element::f16) { + v = std::make_shared(v, ov::element::f16); + }""", + ), + ], + "ggml/src/ggml-openvino/ggml-decoder.h": [ + ( + """ std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) const { + if (is_inp_pos(tensor, op)) { + return "inp_pos"; + }""", + """ // vla.cpp: only collapse ROPE position inputs onto one "inp_pos" parameter + // when the graph really has one. See scripts/patch_ggml_openvino.py. + bool has_multiple_inp_pos() const; + + std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) const { + if (is_inp_pos(tensor, op)) { + return has_multiple_inp_pos() ? std::string(tensor->name) : std::string("inp_pos"); + }""", + ), + ( + " ggml_cgraph * m_cgraph = nullptr;", + " ggml_cgraph * m_cgraph = nullptr;\n" + " mutable int m_multi_inp_pos = -1; // vla.cpp: lazily computed, -1 = unknown", + ), + ], + "ggml/src/ggml-openvino/ggml-decoder.cpp": [ + ( + """ case GGML_OP_ADD: { + if (is_moe_expert_sum_add(node)) {""", + """ case GGML_OP_ADD: { + // vla.cpp: ADD(ADD(.., GEMM), graph-input). Two elementwise ops chained on a + // GEMM; the GPU plugin folds both in as post-ops and loses the second operand. + // op_case 2 = the inner add's input 0 is the GEMM, 3 = its input 1 is. + { + const ggml_tensor * inner = node->src[0]; + const ggml_tensor * other = node->src[1]; + if (inner && other && inner->op == GGML_OP_ADD && other->op == GGML_OP_NONE) { + for (int k = 0; k < 2; k++) { + const ggml_tensor * s = inner->src[k]; + while (s && s->src[0] && + (s->op == GGML_OP_VIEW || s->op == GGML_OP_RESHAPE || s->op == GGML_OP_CONT)) { + s = s->src[0]; + } + if (s && s->op == GGML_OP_MUL_MAT) { + op_case = (k == 0) ? 2 : 3; + } + } + } + } + if (is_moe_expert_sum_add(node)) {""", + ), + ( + """ if (GgmlOvDecoder::is_inp_pos(tensor, op)) { + return "inp_pos"; + }""", + """ if (GgmlOvDecoder::is_inp_pos(tensor, op)) { + // vla.cpp: this free function is the live naming path -- the + // GgmlOvDecoder member of the same intent is unreferenced at b10729. + // Only collapse ROPE position inputs onto one "inp_pos" parameter when + // the graph really has one. See scripts/patch_ggml_openvino.py. + return decoder->has_multiple_inp_pos() ? get_tensor_ov_name(cgraph, tensor) : std::string("inp_pos"); + }""", + ), + ( + """ } else { + // rope'ed query tensor + op_case = 2;""", + """ } else { + // vla.cpp: op_case 2 rewrites the tensor as [n_seq, -1, n_heads, + // head_size] before transposing, which is only correct for + // llama.cpp's rope'd query. It was reached by ANY permute whose + // source is a view of a non-leaf, so a DiT head split -- GR00T + // N1.7's cross-attention V, ggml_permute(view, 1,2,0,3) -- was + // reshaped into a shape that has nothing to do with it and came out + // with its elements rearranged. Require the rope. + const ggml_tensor * prod = node->src[0]; + while (prod && prod->src[0] && + (prod->op == GGML_OP_VIEW || prod->op == GGML_OP_RESHAPE || prod->op == GGML_OP_CONT)) { + prod = prod->src[0]; + } + op_case = (prod && prod->op == GGML_OP_ROPE) ? 2 : 1;""", + ), + ( + """ } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { + op_case = 3;""", + """ // vla.cpp: case 3 is the KV-cache flatten, whose result is always + // [1, n, 1, 1]. Without the ne[0] test it also swallows the kernel + // reshape ggml_conv_2d emits and rewrites it to the wrong shape. + } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1] && node->ne[0] == 1) { + op_case = 3;""", + ), + ( + "int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {", + """bool GgmlOvDecoder::has_multiple_inp_pos() const { + if (m_cgraph == nullptr) { + return false; + } + if (m_multi_inp_pos < 0) { + std::set seen; + for (int i = 0; i < m_cgraph->n_nodes && seen.size() < 2; i++) { + const ggml_tensor * node = m_cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC && node->src[j] != nullptr; j++) { + if (is_inp_pos(node->src[j], node)) { + seen.insert(node->src[j]); + } + } + } + m_multi_inp_pos = seen.size() > 1 ? 1 : 0; + } + return m_multi_inp_pos == 1; +} + +int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {""", + ), + ], + "ggml/src/ggml-openvino/utils.h": [ + ( + "struct decoder_runtime_ctx {", + """// vla.cpp: what naive_compute() reuses across calls on the same graph. Without +// it that path rebuilt the decoder, re-converted the model and called +// compile_model() on every ggml_backend_graph_compute, which dominated runtime. +struct naive_runtime_ctx { + std::mutex mutex; + std::shared_ptr decoder; + std::shared_ptr model; + std::shared_ptr infer_request; +}; + +// vla.cpp: graph_key is {n_nodes, first name, last name}, which two graphs of the +// same size can share. A compiled model is bound to the shapes it was built for, +// so reusing one across a shape change returns another graph's answer with no +// error. Mix the ops and shapes in as well. +inline uint64_t naive_graph_sig(const ggml_cgraph * cgraph) { + uint64_t h = 1469598103934665603ull; + auto mix = [&h](uint64_t v) { h = (h ^ v) * 1099511628211ull; }; + for (int i = 0; i < cgraph->n_nodes; i++) { + const ggml_tensor * node = cgraph->nodes[i]; + mix((uint64_t) node->op); + mix((uint64_t) node->type); + for (int d = 0; d < GGML_MAX_DIMS; d++) { + mix((uint64_t) node->ne[d]); + } + for (int s = 0; s < GGML_MAX_SRC; s++) { + const ggml_tensor * src = node->src[s]; + mix(src ? (uint64_t) src->type + 1 : 0ull); + for (int d = 0; src && d < GGML_MAX_DIMS; d++) { + mix((uint64_t) src->ne[d]); + } + } + } + return h; +} + +struct naive_key { + graph_key base; + uint64_t sig; + + naive_key(const ggml_cgraph * cgraph) : base(cgraph), sig(naive_graph_sig(cgraph)) {} + + bool operator==(const naive_key & other) const { return sig == other.sig && base == other.base; } +}; + +struct naive_key_hash { + size_t operator()(const naive_key & key) const { + size_t h = graph_key_hash{}(key.base); + return h ^ (std::hash{}(key.sig) + 0x9e3779b9 + (h << 6) + (h >> 2)); + } +}; + +struct decoder_runtime_ctx {""", + ), + ( + " std::unordered_map, graph_key_hash> decoder_cache;", + " std::unordered_map, graph_key_hash> decoder_cache;\n" + " std::unordered_map, naive_key_hash> naive_cache;", + ), + ( + """ decoder_cache.clear(); + infer_request_cache.clear();""", + """ decoder_cache.clear(); + naive_cache.clear(); + infer_request_cache.clear();""", + ), + ( + """enum ggml_status naive_compute(struct ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config);""", + """enum ggml_status naive_compute(struct ggml_cgraph * cgraph, + ov::Core & core, + const std::string & device, + const ov::AnyMap & config, + std::shared_ptr r_ctx);""", + ), + ], + "ggml/src/ggml-openvino/utils.cpp": [ + ( + """ if (!model_is_splitted) { + return naive_compute(cgraph, core, device, config); + }""", + """ if (!model_is_splitted) { + return naive_compute(cgraph, core, device, config, r_ctx); + }""", + ), + ( + """ if (is_naive(cgraph)) { + return naive_compute(cgraph, core, device, config); + }""", + """ if (is_naive(cgraph)) { + return naive_compute(cgraph, core, device, config, r_ctx); + }""", + ), + ( + NAIVE_COMPUTE_OLD, + NAIVE_COMPUTE_NEW, + ), + ( + """bool is_naive(ggml_cgraph * cgraph) { + constexpr int naive_graph_size_threshold = 20;""", + """bool is_naive(ggml_cgraph * cgraph) { + // vla.cpp: the literal translation path suits any graph that is not a + // decoder-only LLM, so let the caller raise the bar it is chosen under. + // Junk parses to 0 under atoi, which would silently send every graph down + // the LLM builder, so reject anything that is not a whole positive number. + static const int naive_graph_size_threshold = [] { + const char * env = getenv("GGML_OPENVINO_NAIVE_GRAPH_SIZE"); + if (env == nullptr || *env == '\\0') { + return 20; + } + char * end = nullptr; + const long val = strtol(env, &end, 10); + const int n = (int) val; + if (*end != '\\0' || val <= 0 || (long) n != val) { + GGML_LOG_WARN("GGML OpenVINO Backend: ignoring GGML_OPENVINO_NAIVE_GRAPH_SIZE='%s'\\n", env); + return 20; + } + return n; + }();""", + ), + ], +} + + +# A duplicate key in EDITS silently drops the earlier entry's hunks -- Python keeps +# the last one and nothing complains. That happened once and took the whole Intel +# OpenCL fix out of the patch, so compare the literal keys against the dict. +_keys = re.findall(r'^ "([^"]+)": \[$', pathlib.Path(__file__).read_text(), re.M) +if len(_keys) != len(EDITS): + _dup = sorted({k for k in _keys if _keys.count(k) > 1}) + sys.exit(f"patch_ggml_openvino: EDITS has {len(_keys)} literal keys but {len(EDITS)} entries; " + f"duplicate key(s): {_dup}") + + +def main() -> int: + root = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else ".") + + pending = [] + for rel, edits in EDITS.items(): + src = root / rel + if not src.is_file(): + print(f"patch_ggml_openvino: {src} not found", file=sys.stderr) + return 1 + original = src.read_text() + text = original + for anchor, replacement in edits: + # Per hunk, not per file. The hunk list grows between commits, so a + # tree patched by an older checkout still needs the newer ones and a + # file-wide marker would skip them, leaving a build that runs and is + # quietly wrong. + if replacement in text: + continue + n = text.count(anchor) + if n != 1: + print(f"patch_ggml_openvino: anchor matched {n} times in {rel}, expected 1:\n{anchor}\n" + f"A tree patched by an older checkout reads like this. Delete the fetched " + f"llama.cpp (rm -rf /_deps) and reconfigure. If the tree is clean, the " + f"anchor no longer matches VLA_LLAMA_TAG and needs re-targeting.", + file=sys.stderr) + return 1 + text = text.replace(anchor, replacement, 1) + if text != original: + pending.append((src, text, rel)) + + # Every anchor matched, so nothing was half-written on the way here. + for src, text, rel in pending: + src.write_text(text) + print(f"patch_ggml_openvino: patched {rel}") + if not pending: + print("patch_ggml_openvino: already up to date") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/print_versions.sh b/scripts/print_versions.sh index 4681af5..678dfaa 100644 --- a/scripts/print_versions.sh +++ b/scripts/print_versions.sh @@ -79,12 +79,12 @@ echo "- repo HEAD: \`${VLA_HEAD}\` (\`${VLA_DESC}\`)" # ---- llama.cpp ---- echo -echo "### llama.cpp (third_party)" +echo "### llama.cpp (fetched)" echo LLAMA_HEAD=$(git_head_or_dash "$LLAMA_DIR") LLAMA_DESC=$(git_describe_or_dash "$LLAMA_DIR") echo "- HEAD: \`${LLAMA_HEAD}\` (\`${LLAMA_DESC}\`)" -LLAMA_TAG=$(grep -m1 'GIT_TAG' "$ROOT/CMakeLists.txt" | grep -oE 'b[0-9]+' || echo '?') +LLAMA_TAG=$("$ROOT/scripts/llama_tag.sh" 2>/dev/null || echo '?') echo "- expected pinned tag (from \`CMakeLists.txt\`): \`${LLAMA_TAG}\`" # ---- GGUFs ---- diff --git a/scripts/upstream_split.py b/scripts/upstream_split.py new file mode 100755 index 0000000..2b328f7 --- /dev/null +++ b/scripts/upstream_split.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Split scripts/patch_ggml_openvino.py into one llama.cpp branch per upstream PR. + +Every hunk in the patch script is a generic ggml-openvino fix, so it belongs +upstream rather than in a configure-time rewrite here. This regroups the hunks +into per-PR commits on a clone of llama.cpp master, which is what +docs/UPSTREAMING.md tracks. It only writes to that clone. + + git clone --depth 1 https://github.com/ggml-org/llama.cpp ~/llama.cpp-upstream + python3 scripts/upstream_split.py [] + +Every hunk must land in exactly one PR; the tail of the output says so. +""" +import importlib.util, os, pathlib, subprocess, sys + +REPO = pathlib.Path(sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/llama.cpp-upstream")) +HERE = pathlib.Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location("p", HERE / "patch_ggml_openvino.py") +m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +E = m.EDITS +D = "ggml/src/ggml-openvino/" + + +def H(rel, key): + """Address a hunk by a unique substring of its anchor, not by position. + + These used to be plain indices, and adding a hunk to the front of a file's + list silently handed every later hunk to the wrong branch -- the tail + coverage count still read 29/29, because each index was still used once. + A key that stops matching fails here instead of committing the wrong diff. + """ + hits = [i for i, (anchor, _) in enumerate(E[rel]) if key in anchor] + if len(hits) != 1: + sys.exit(f"upstream_split: key {key!r} matched {len(hits)} hunks in {rel}, expected 1") + return rel, hits[0] + + +# branch -> (subject, body, [H(file, anchor-substring), ...]) +PRS = [ + ("openvino-naive-cache", + "openvino: cache the compiled model on the naive path", + "The dynamic and static paths keep a graph_key-indexed cache of the decoder and\n" + "the compiled infer request. The naive path has none, so every\n" + "ggml_backend_graph_compute rebuilds the decoder, re-converts the model and\n" + "calls compile_model() again. On a graph that is not a decoder-only LLM that is\n" + "the dominant cost: a 512px SmolVLA vision tower plus action expert goes from\n" + "22.7 s to 1.4 s per prediction on the CPU plugin.\n\n" + "A hit rebinds the cached decoder through the existing update_io(), the same way\n" + "the dynamic path handles freshly built tensors.\n\n" + "The cache is keyed on naive_key rather than graph_key. graph_key is n_nodes plus\n" + "the first and last node name, which two graphs of the same size can share, and a\n" + "compiled model is bound to the shapes it was built for, so a collision returns\n" + "another graph's answer with no error. naive_key mixes in every node's op, type\n" + "and shape. The map is bounded and flushed when full.", + [H(D+"utils.h","struct decoder_runtime_ctx"),H(D+"utils.h","graph_key_hash> decoder_cache"), + H(D+"utils.h","decoder_cache.clear()"),H(D+"utils.h","enum ggml_status naive_compute"), + H(D+"utils.cpp","if (!model_is_splitted)"),H(D+"utils.cpp","if (is_naive(cgraph))"), + H(D+"utils.cpp","enum ggml_status naive_compute")]), + + ("openvino-naive-graph-size-env", + "openvino: make the naive-path graph-size threshold settable", + "Graphs under 20 nodes bypass the LLM decoder and translate literally, with\n" + "static shapes and no KV-cache inference. That literal path is the one that fits\n" + "a graph which is not a decoder-only transformer, but such a graph is routinely\n" + "far larger than 20 nodes: a ViT tower is around 450.\n\n" + "Expose the constant as GGML_OPENVINO_NAIVE_GRAPH_SIZE. Parsed with strtol and\n" + "rejected loudly if it is not a whole positive number, because atoi turns junk\n" + "into 0 and that would send every graph down the LLM builder with nothing said.", + [H(D+"utils.cpp","bool is_naive(ggml_cgraph")]), + + ("openvino-gelu-modes", + "openvino: map GELU to tanh and add GELU_ERF", + "ggml has two GELUs: GGML_UNARY_OP_GELU is the tanh approximation and\n" + "GGML_UNARY_OP_GELU_ERF is the exact one. The table maps GELU onto ov's Gelu,\n" + "which defaults to erf, and has no entry for GELU_ERF at all. So the tanh op is\n" + "computed as erf, and a graph using the erf op cannot run.\n\n" + "Small per node, but a vision tower has dozens and it compounds: on a ggml graph\n" + "with a ViT encoder, fixing the mode moved two models from visibly wrong output\n" + "to within 1.3e-3 of the CPU-backend reference.", + [H(D+"openvino/op_table.cpp","namespace ov {"),H(D+"openvino/op_table.cpp","{\"GGML_UNARY_OP_GELU\",")]), + + ("openvino-multiple-inp-pos", + "openvino: stop distinct position inputs aliasing each other", + "Every tensor feeding a ROPE's second input is renamed to one graph parameter\n" + "called inp_pos, because a llama.cpp graph has exactly one. add_rope_sin_cos()\n" + "then builds a single shared sin/cos table from it.\n\n" + "A graph with several position tensors -- a prefill, a full and a rebased one --\n" + "has them alias each other, and every ROPE takes the table built from whichever\n" + "won. Shape inference then fails:\n\n" + " Multiply (Split[1]:f32[1,113,5,32], Multiply[0]:f32[1,50,1,32])\n" + " Argument shapes are inconsistent.\n\n" + "When the graph has more than one, keep each tensor's own name. Nothing is then\n" + "called inp_pos, so translate_rope() builds sin/cos per op from its own position\n" + "input. Single-position graphs are untouched.\n\n" + "Guard the free get_tensor_graph_input_ov_name() as well as the GgmlOvDecoder\n" + "member: the free function is the one compute_model_inputs() and\n" + "set_input_output() actually call, and the member currently has no callers.", + [H(D+"ggml-decoder.h","get_graph_input_ov_name"),H(D+"ggml-decoder.h","m_cgraph = nullptr"), + H(D+"ggml-decoder.cpp","is_inp_pos(tensor, op)"),H(D+"ggml-decoder.cpp","compute_op_case(const ggml_tensor")]), + + ("openvino-reshape-op-case", + "openvino: narrow the RESHAPE op_case 3 guard", + "Case 3 is the KV-cache flatten, [512,1024,1,1] -> [1,524288,1,1], and it emits a\n" + "shape with -1 in dim 2 and 1 in dim 3. Its guard only tests\n" + "src->ne[0]*ne[1]*ne[2] == node->ne[1], which also matches the kernel reshape\n" + "inside ggml_conv_2d ([16,16,3,768] -> [768,768]) and rewrites it to the wrong\n" + "shape. The real case always has node->ne[0] == 1; requiring that sends the conv\n" + "kernel to case 6, the plain reshape.", + [H(D+"ggml-decoder.cpp","== node->ne[1]")]), + + ("openvino-gemm-double-eltwise", + "openvino: do not stack two elementwise adds on a GEMM", + "The GPU plugin folds elementwise ops into the preceding GEMM as post-ops. Given\n" + "ADD(ADD(residual, GEMM), graph_input) it folds both and the second operand is\n" + "silently lost: the result equals the inner add, as though the outer one never\n" + "ran. Nothing is logged.\n\n" + "A decoder-only LLM graph never builds that chain - one residual add per\n" + "sub-block - but a graph that adds a vision tower's features on top of an FFN\n" + "residual does, and three such models produced badly wrong output on GPU while\n" + "matching the CPU plugin to 1e-4.\n\n" + "Addition is associative, so re-hang the outer add on the inner one's non-GEMM\n" + "operand; the GEMM keeps exactly one post-op. op_case 2/3 records which operand\n" + "of the inner add is the GEMM, since the order is not fixed.\n\n" + "Same fusion path as the broadcast-DIV defect already handled in supports_op.", + [H(D+"ggml-decoder.cpp","case GGML_OP_ADD: {"), + H(D+"openvino/op/add.cpp","auto input_0 = process_view_input_new(context, 0);")]), + + ("openvino-permute-op-case", + "openvino: require a ROPE before taking PERMUTE op_case 2", + "op_case 2 rewrites the tensor as [n_seq, -1, n_heads, head_size] and only then\n" + "transposes, which is correct for llama.cpp's rope'd query and nothing else. The\n" + "classifier reaches it for ANY permute whose source is a view of a non-leaf.\n\n" + "A DiT cross-attention V built as ggml_permute(view, 1,2,0,3) over a fused KV\n" + "projection is therefore reshaped into a shape unrelated to it and comes out with\n" + "its elements rearranged, while a sibling K using permute(0,2,1,3) survives the\n" + "same rewrite -- 139% wrong against 0.04%, with no error reported.\n\n" + "Walk the view/reshape/cont chain and require a ROPE at the end; every other\n" + "permute falls to op_case 1, the plain transpose.", + [H(D+"ggml-decoder.cpp","rope'ed query tensor")]), + + ("openvino-sdpa-kv-f16", + "openvino: convert K/V to F16 alongside Q in flash_attn_ext", + "The translator converts Q, the mask and the scale to F16 because llama.cpp's KV\n" + "cache already is. A caller that keeps K/V in F32 hits OpenVINO's SDPA rejecting\n" + "mixed input types (\"Mixed input types are not supported\"). Converting K/V too\n" + "matches the precision the translator has already chosen for the other operands.", + [H(D+"openvino/op/flash_attn_ext.cpp","q_f32, ov::element::f16")]), + + ("openvino-view-input-rank", + "openvino: give a folded weight its full rank before slicing", + "A ggml tensor that is 2-D folds in as a rank-2 ov constant, which is what a GEMM\n" + "operand wants, but process_view_input_new() indexes a viewed tensor at its full\n" + "ggml rank, so the slice axis lands outside it:\n\n" + " Slice (Constant blk.0.attn_in.weight[0]:bf16[2688,896], ...)\n" + " Axis 2 out of the tensor rank range [-2, 1].\n\n" + "Reached by viewing Q, K and V out of one fused attn_in weight. Left-pad the\n" + "input with leading 1s, which is the shape ggml gave it anyway.", + [H(D+"openvino/utils.cpp","openvino/op/transpose.hpp"),H(D+"openvino/utils.cpp","get_view_input_size(input_index)")]), + + ("openvino-concat-rank", + "openvino: align CONCAT input ranks", + "The same rank-2 folded constants reach CONCAT, which unlike the broadcasting\n" + "elementwise ops needs both inputs at the graph's rank or the axis falls outside\n" + "them. Hit by concatenating a CLS weight onto 4-D patch embeddings, and by\n" + "concatenating a precomputed time tile onto a 4-D activation. Left-pad the\n" + "shorter input with leading 1s before picking the axis.", + [H(D+"openvino/op/concat.cpp","openvino/op/concat.hpp"),H(D+"openvino/op/concat.cpp","#include "), + H(D+"openvino/op/concat.cpp","rank - 1 - ggml_dim")]), + + ("openvino-imrope-sections", + "openvino: bound the interleaved-mrope sector cycle by sections", + "ggml's IMROPE cycles t/h/w by sector % 3, but only while the sector is inside\n" + "3 * sections[k]; past that it falls through to the fourth position stream\n" + "(ggml_rope_cache_init in ggml/src/ggml-cpu/ops.cpp). The translator cycled\n" + "unconditionally, so with sections {24,20,20,0} and n_dims 128, sectors 61 and 62\n" + "took h and w instead of e.\n\n" + "No measurable output change on the graphs tested, because the fourth stream\n" + "happened to carry the same positions as the first. Submitted because it removes\n" + "a divergence from the ggml reference, not because a measurement demanded it.", + [H(D+"openvino/utils.cpp","#include "),H(D+"openvino/utils.cpp","gather_indices(n_dims_half)")]), + + ("openvino-gpu-precision-env", + "openvino: expose the GPU plugin's inference precision", + "The GPU plugin computes in f16 unless told otherwise, which is most of why it is\n" + "fast, and for nearly every graph that is the right trade.\n\n" + "It is not the right trade for a graph carrying a long serial chain, such as a\n" + "flow-matching denoise loop unrolled inside a single graph: the error compounds\n" + "across every step with nothing to reset it, and a saturating output channel can\n" + "then cross its threshold in the wrong place. One such model lands 4e-2 from an\n" + "F32 reference on GPU and 6.5e-5 with this set to f32.\n\n" + "Exposed rather than forced -- f32 costs roughly 3x -- and defaulted to f16, so\n" + "existing behaviour is unchanged unless the variable is set.", + [H(D+"ggml-openvino-extra.cpp","GGML_OPENVINO_LOG_UNSUPPORTED_OPS"), + H(D+"ggml-openvino-extra.cpp","} else if (cache_dir && strlen(cache_dir) > 0) {")]), + + ("openvino-intel-opencl-platform", + "openvino: select the Intel OpenCL platform for the GPU remote context", + "GGML_OPENVINO_DEVICE=GPU builds an OpenVINO remote context on an OpenCL queue and\n" + "takes the first platform the ICD loader reports. With more than one runtime\n" + "installed (an NVIDIA card beside the Intel iGPU, POCL, Rusticl) that is whichever\n" + "/etc/OpenCL/vendors/*.icd sorted first, and the GPU plugin only accepts an Intel\n" + "context. It aborts at startup with \"Incompatible OpenCL runtime: program is not\n" + "in expected ELF format\".\n\n" + "Select by CL_PLATFORM_VENDOR instead. Single-runtime hosts are unaffected.", + [H(D+"ggml-openvino-extra.cpp","#include "),H(D+"ggml-openvino-extra.cpp","ggml_openvino_device_config::init"), + H(D+"ggml-openvino-extra.cpp","cl_int err;"),H(D+"ggml-openvino-extra.cpp","clEnqueueMemFillINTEL_fn"), + H(D+"ggml-openvino-extra.cpp","clEnqueueMemcpyINTEL_fn")]), +] + +def git(*a): + r = subprocess.run(["git", "-C", str(REPO), *a], capture_output=True, text=True) + if r.returncode: sys.exit(f"git {' '.join(a)} failed:\n{r.stderr}") + return r.stdout + +base = git("rev-parse", "HEAD").strip() +used = set() +for branch, subject, body, hunks in PRS: + git("checkout", "-q", "-B", branch, base) + for rel, idx in hunks: + used.add((rel, idx)) + anchor, repl = E[rel][idx] + # The marker is vla.cpp provenance; upstream comments should not carry it. + repl = repl.replace('vla.cpp: ', '') + f = REPO / rel + t = f.read_text() + if repl in t: continue + if t.count(anchor) != 1: + sys.exit(f"{branch}: anchor {idx} in {rel} matched {t.count(anchor)} times") + f.write_text(t.replace(anchor, repl, 1)) + git("add", "-A") + git("commit", "-q", "-m", subject, "-m", body) + print(f"{branch:38s} {git('rev-parse','--short','HEAD').strip()} {len(hunks)} hunk(s)") + +git("checkout", "-q", base) +total = sum(len(v) for v in E.values()) +missing = [(r, i) for r, v in E.items() for i in range(len(v)) if (r, i) not in used] +print(f"\n{len(used)}/{total} hunks assigned to {len(PRS)} branches") +if missing: print("UNASSIGNED:", missing) diff --git a/src/backend.h b/src/backend.h index 33377d4..9223fc5 100644 --- a/src/backend.h +++ b/src/backend.h @@ -21,10 +21,10 @@ * instead; the ladder lives here once. * * Exactly one accelerator is compiled in, picked by the CMake flag that was - * used (`GGML_CUDA` / `GGML_SYCL` / `GGML_METAL`). There is no per-op CPU - * fallback: the core drives a single backend through `gallocr` rather than a - * scheduler, so an arch that hits an op the backend does not implement asserts - * at predict time instead of silently limping. + * used (`GGML_CUDA` / `GGML_SYCL` / `GGML_METAL` / `GGML_OPENVINO`). There is no + * per-op CPU fallback: the core drives a single backend through `gallocr` rather + * than a scheduler, so an arch that hits an op the backend does not implement + * asserts at predict time instead of silently limping. */ #pragma once @@ -42,26 +42,38 @@ #ifdef GGML_USE_METAL #include "ggml-metal.h" #endif +#ifdef GGML_USE_OPENVINO +#include "ggml-openvino.h" +#endif #include +#include #include -#ifdef GGML_USE_SYCL +#ifdef GGML_USE_OPENVINO +#include +#include +#endif +#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) #include // setenv / _putenv_s #include #endif namespace vla { -#ifdef GGML_USE_SYCL +#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) // setenv is POSIX. _putenv_s has no "do not overwrite" mode, so check first. +// Empty counts as unset; an empty KEY= in a compose file is not a choice. inline void setenv_default(const char * key, const char * val) { #ifdef _WIN32 size_t len = 0; - if (getenv_s(&len, nullptr, 0, key) == 0 && len > 0) + if (getenv_s(&len, nullptr, 0, key) == 0 && len > 1) // len counts the NUL return; _putenv_s(key, val); #else - setenv(key, val, /*overwrite=*/0); + const char * cur = std::getenv(key); + if (cur && *cur) + return; + setenv(key, val, /*overwrite=*/1); #endif } #endif @@ -76,6 +88,69 @@ struct Backend { bool is_cuda = false; }; +/** + * @brief Give every tensor in a built graph a name unique within that graph. + * + * ggml derives a name for a result from its source -- `ggml_reshape_2d` of an + * unnamed tensor is called " (reshaped)" -- so a graph whose intermediates were + * never named ends up with many tensors sharing one name. That is legal ggml, + * and llama.cpp never trips over it because it labels every node it builds. + * + * ggml-openvino keys its translation map on those names: two nodes with the same + * name silently become one, and the graph it hands OpenVINO wires the wrong + * tensor into the next op. Renaming duplicates in place before compute is enough + * to keep them apart, and only the OpenVINO build pays for it -- elsewhere this + * compiles to nothing, so backend logs and profiles keep the names ggml chose. + * + * @param gf Graph to relabel, already built. + */ +inline void graph_unique_names([[maybe_unused]] ggml_cgraph * gf) { +#ifdef GGML_USE_OPENVINO + // Leafs cannot collide: ggml names an unnamed one "leaf_" as it walks the + // graph, and a named one came from the checkpoint. Only results carry a name + // derived from their source, so only nodes are checked. + const int n = ggml_graph_n_nodes(gf); + + std::unordered_set seen; + seen.reserve((size_t) n); + + char buf[GGML_MAX_NAME]; + for (int i = 0; i < n; ++i) { + ggml_tensor * t = ggml_graph_node(gf, i); + if (seen.insert(ggml_get_name(t)).second) continue; + // Local buffer: ggml_format_name would pass t->name as both destination + // and "%s" source, and glibc empties it. Index leads so truncation eats + // the tail, not the distinguisher. k steps by n to stay out of range. + for (int k = i; ; k += n) { + std::snprintf(buf, sizeof(buf), "%d#%s", k, ggml_get_name(t)); + if (seen.insert(buf).second) break; + } + ggml_set_name(t, buf); + } +#endif +} + +/** + * @brief Allocate an arch's weights and tag the buffer as holding weights. + * + * `ggml_backend_alloc_ctx_tensors` leaves the buffer on the default + * `GGML_BACKEND_BUFFER_USAGE_ANY`, which backends read as "this is not weights". + * ggml-openvino takes it literally and classifies every ANY tensor as a KV + * cache, giving it a dynamic sequence dimension; the first translator that asks + * a weight for its static shape then throws `to_shape was called on a dynamic + * shape`. Tagging the buffer is what llama.cpp does with its own weights, and it + * is what lets the OpenVINO frontend fold them in as constants. + * + * @return The weight buffer, or null if the allocation failed (OOM). + */ +inline ggml_backend_buffer_t alloc_weights(ggml_context * ctx, ggml_backend_t backend) { + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (buf) { + ggml_backend_buffer_set_usage(buf, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + } + return buf; +} + /// GPU ordinal for CUDA and SYCL; `VLA_DEVICE` overrides. Junk is rejected, not /// silently read as device 0. inline int backend_device_index() { @@ -153,6 +228,71 @@ inline Backend backend_init(const char * tag, int n_threads) { std::fprintf(stderr, "%s: ggml_backend_metal_init failed; falling back to CPU\n", tag); } } +#elif defined(GGML_USE_OPENVINO) + { + // ggml-openvino translates a graph under 20 nodes literally and anything + // larger through a decoder-only-LLM model builder that infers a KV cache + // and one position input. No vla.cpp graph is that shape and all of them + // are far larger, so raise the bar the literal path is chosen under; + // scripts/patch_ggml_openvino.py is what makes the threshold settable. + // Only a default: an explicit setting wins. The patched reader latches it + // on the first graph_compute, so here is early enough. call_once because + // a concurrent model_load would race on the environment. + static std::once_flag naive_once; + std::call_once(naive_once, [] { setenv_default("GGML_OPENVINO_NAIVE_GRAPH_SIZE", "1000000"); }); + + // pi0 runs its whole 10-step denoise loop inside a single graph, so the + // GPU plugin's F16 arithmetic compounds across every step with nothing to + // reset it. On the continuous action dims that shows up as ~4e-2 against + // an F32 reference, and it is enough to move the bistable gripper channel + // across its threshold a step early -- which reads as a 1.7 error on a + // metric, and as the gripper closing late on a robot. Asking the GPU for + // F32 puts it back at 6.5e-5, and costs about 3x (383 ms -> 1170 ms). + // Only pi0 needs it: every other arch is inside the bar on the GPU at F16. + // A default, so GGML_OPENVINO_GPU_PRECISION=f16 still wins. + // Exact match: `tag` is the log prefix, and "vla(pi05)" contains "vla(pi0)", + // so anything looser would drag pi0.5 in too -- it does not need this. + if (tag && std::strcmp(tag, "vla(pi0)") == 0) { + setenv_default("GGML_OPENVINO_GPU_PRECISION", "f32"); + } + + // ggml exposes OpenVINO as a single device, so VLA_DEVICE does not apply: + // the target is chosen by name through GGML_OPENVINO_DEVICE (CPU / GPU / + // NPU) and resolved inside ggml. + // + // OpenVINO's blob cache reloads a graph that computes the wrong thing: + // cold run correct, next run wrong, nothing logged. A warning is no use + // when stderr goes nowhere, so clear it and make opting back in explicit. + if (const char * cd = std::getenv("GGML_OPENVINO_CACHE_DIR"); cd && *cd) { + const char * allow = std::getenv("VLA_ALLOW_OV_CACHE"); + const bool keep = allow && allow[0] == '1' && allow[1] == '\0'; + std::fprintf(stderr, + "%s: WARNING GGML_OPENVINO_CACHE_DIR is set. Reloading cached blobs has been\n" + "%s: seen to produce silently incorrect actions on the GPU plugin.\n" + "%s: %s See docs/backend/ov.md.\n", + tag, tag, tag, + keep ? "Kept: VLA_ALLOW_OV_CACHE=1." + : "Ignoring it; set VLA_ALLOW_OV_CACHE=1 to keep it."); + if (!keep) { +#ifdef _WIN32 + _putenv_s("GGML_OPENVINO_CACHE_DIR", ""); +#else + unsetenv("GGML_OPENVINO_CACHE_DIR"); +#endif + } + } + + // ggml falls back to CPU silently when the requested device is missing, + // so this line is the request. ggml logs what actually ran. + const char * want = std::getenv("GGML_OPENVINO_DEVICE"); + b.handle = ggml_backend_openvino_init(0); + if (b.handle) { + std::printf("%s: backend = OPENVINO (asked for %s, see ggml's \"using device\" line)\n", + tag, (want && *want) ? want : "CPU"); + } else { + std::fprintf(stderr, "%s: ggml_backend_openvino_init failed; falling back to CPU\n", tag); + } + } #endif if (!b.handle) { diff --git a/src/loader.cpp b/src/loader.cpp index 9a67f05..8e231c0 100644 --- a/src/loader.cpp +++ b/src/loader.cpp @@ -14,6 +14,8 @@ #include "loader.h" +#include "backend.h" + #include #include #include @@ -134,9 +136,9 @@ bool WeightLoader::upload(ggml_backend_t backend, ggml_backend_buffer_t * out_bu return false; } - ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx_, backend); + ggml_backend_buffer_t buf = alloc_weights(ctx_, backend); if (!buf) { - std::fprintf(stderr, "vla(%s): ggml_backend_alloc_ctx_tensors failed (OOM?)\n", arch_); + std::fprintf(stderr, "vla(%s): alloc_weights failed (OOM?)\n", arch_); return false; } *out_buf = buf; @@ -160,21 +162,21 @@ bool WeightLoader::upload(ggml_backend_t backend, ggml_backend_buffer_t * out_bu } for (const Fused & f : fused_) { - std::vector buf; + std::vector parts; for (const std::string & s : f.srcs) { std::vector b = g_.read_convert(s.c_str(), f.dst->type); if (b.empty()) { std::fprintf(stderr, "vla(%s): fused fill: read %s failed\n", arch_, s.c_str()); return false; } - buf.insert(buf.end(), b.begin(), b.end()); + parts.insert(parts.end(), b.begin(), b.end()); } - if (buf.size() != ggml_nbytes(f.dst)) { + if (parts.size() != ggml_nbytes(f.dst)) { std::fprintf(stderr, "vla(%s): fused fill: %s size %zu vs %zu\n", - arch_, ggml_get_name(f.dst), buf.size(), ggml_nbytes(f.dst)); + arch_, ggml_get_name(f.dst), parts.size(), ggml_nbytes(f.dst)); return false; } - ggml_backend_tensor_set(f.dst, buf.data(), 0, buf.size()); + ggml_backend_tensor_set(f.dst, parts.data(), 0, parts.size()); } return true; } diff --git a/src/model.cpp b/src/model.cpp index b700ec3..f379a9c 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -156,7 +156,9 @@ bool detect_arch_safetensors(const std::string& path, Arch* out) { return false; uint64_t header_size = 0; f.read(reinterpret_cast(&header_size), sizeof(header_size)); - if (!f || header_size == 0 || header_size > (1u << 28)) + // The probe only substring-searches for a namespace prefix, so it never + // needs more than a real header: the largest in tree is a couple of MB. + if (!f || header_size == 0 || header_size > (1u << 24)) return false; std::string header(header_size, '\0'); f.read(header.data(), header_size); diff --git a/src/models/bitvla.cpp b/src/models/bitvla.cpp index a115b5e..71df94f 100644 --- a/src/models/bitvla.cpp +++ b/src/models/bitvla.cpp @@ -24,6 +24,7 @@ #include "ggml-cuda.h" #endif #include "gguf.h" +#include "backend.h" #include "gguf_reader.h" #include "scratch_ctx.h" @@ -220,6 +221,9 @@ ggml_tensor * build_lm_layer(ggml_context * C, const BitvlaModelArch & m, const ggml_tensor * K = ggml_cont(C, ggml_permute(C, kR, 0, 2, 1, 3)); ggml_tensor * V = ggml_cont(C, ggml_permute(C, v3, 1, 2, 0, 3)); ggml_tensor * kq = ggml_mul_mat(C, K, Q); ggml_mul_mat_set_prec(kq, GGML_PREC_F32); + // Unmasked on purpose, same as openvla_oft: BitVLA is fine-tuned with + // OpenVLA-OFT's recipe, which swaps the causal mask for a bidirectional one + // so the action chunk decodes in a single pass. ggml_tensor * att= ggml_soft_max_ext(C, kq, nullptr, scale, 0.0f); ggml_tensor * kqv= ggml_mul_mat(C, V, att); ggml_tensor * mer= ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, kqv, 0, 2, 1, 3)), hq, seq); @@ -1184,6 +1188,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_build_forward_expand(gf, mm2); if (!vision_scratch.alloc(backend, gf)) { std::fprintf(stderr, "vla(bitvla): gallocr_alloc_graph failed (view %lld)\n", (long long) v); return {}; } ggml_backend_tensor_set(x_in, patches.data(), 0, ggml_nbytes(x_in)); + graph_unique_names(gf); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): vision graph compute failed (view %lld)\n", (long long) v); return {}; @@ -1224,6 +1229,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_build_forward_expand(gf, out); if (!proprio_scratch.alloc(backend, gf)) { std::fprintf(stderr, "vla(bitvla): gallocr failed (proprio)\n"); return {}; } ggml_backend_tensor_set(x_in, state_host.data(), 0, ggml_nbytes(x_in)); + graph_unique_names(gf); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): proprio compute failed\n"); return {}; } ggml_backend_tensor_get(out, proprio_embed_host.data(), 0, (size_t) hidden_l * sizeof(float)); } @@ -1265,6 +1271,14 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { std::fprintf(stderr, "vla(bitvla): seq=%lld > lm_max_pos=%lld\n", (long long) seq, (long long) lm_max_pos); return {}; } + // Both LM paths index the action slots as seq-2-n_action+i and neither + // ggml_get_rows nor the CUDA gather bound-checks, so a short sequence would + // read out of bounds and come back as plausible hidden states. + if (seq < n_action+2) { + std::fprintf(stderr, "vla(bitvla): seq=%lld too short for %lld action slots\n", + (long long) seq, (long long) n_action); + return {}; + } std::vector inputs_embeds((size_t) seq * hidden_l, 0.0f); @@ -1381,6 +1395,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { aids[i] = (int32_t) (seq-2-n_action+i); ggml_backend_tensor_set(action_ids, aids.data(), 0, ggml_nbytes(action_ids)); + graph_unique_names(gf); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): lm prefill compute failed\n"); return {}; } ggml_backend_tensor_get(action_hidden, last_hidden_at_actions.data(), 0, (size_t) n_action * hidden_l * sizeof(float)); } @@ -1430,6 +1445,7 @@ std::vector BitvlaModelArch::predict(const Inputs& in) { ggml_build_forward_expand(gf, y); if (!head_scratch.alloc(backend, gf)) { std::fprintf(stderr, "vla(bitvla): gallocr failed (action_head)\n"); return {}; } ggml_backend_tensor_set(x, last_hidden_at_actions.data(), 0, ggml_nbytes(x)); + graph_unique_names(gf); if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(bitvla): action_head compute failed\n"); return {}; } ggml_backend_tensor_get(y, normalized_actions.data(), 0, (size_t) chunk * action_dim * sizeof(float)); } diff --git a/src/models/dit_common.h b/src/models/dit_common.h deleted file mode 100644 index ab6047b..0000000 --- a/src/models/dit_common.h +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2026 VinRobotics -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// DiT head pieces shared verbatim by GR00T N1.5/N1.6/N1.7 and VLA-JEPA. dit_kv -// and build_dit_block take a per-arch struct and stay in their own files. - -#pragma once - -#include "ggml.h" - -#include -#include -#include -#include - -namespace vla { - -// Row id of a stacked [out, in, n_embodiment] weight, applied to x. -inline ggml_tensor * cat_linear(ggml_context * C, ggml_tensor * W3d, ggml_tensor * b2d, int64_t id, ggml_tensor * x) { - const int64_t out = W3d->ne[0], in = W3d->ne[1]; - ggml_tensor * W_id = ggml_view_2d(C, W3d, out, in, W3d->nb[1], (size_t) id * W3d->nb[2]); - ggml_tensor * y = ggml_mul_mat(C, ggml_cont(C, ggml_transpose(C, W_id)), x); - return ggml_add(C, y, ggml_view_1d(C, b2d, out, (size_t) id * b2d->nb[1])); -} - -// Per-block AdaLN. The conditioning vector is (scale, shift) in that order; the -// final projection layer in each arch uses (shift, scale) instead. -inline ggml_tensor * adaln(ggml_context * C, ggml_tensor * x, ggml_tensor * temb, ggml_tensor * lw, ggml_tensor * lb, int64_t dim, float eps) { - ggml_tensor * cond = ggml_add(C, ggml_mul_mat(C, lw, ggml_silu(C, temb)), lb); - ggml_tensor * sc = ggml_view_1d(C, cond, dim, 0), * sh = ggml_view_1d(C, cond, dim, (size_t) dim * sizeof(float)); - ggml_tensor * xn = ggml_norm(C, x, eps); - return ggml_add(C, ggml_add(C, xn, ggml_mul(C, xn, sc)), sh); -} - -// cos first, then sin. Opposite order to action_sinusoid; both match the -// reference and are pinned by tests/test_dit_common.cpp. -inline void timesteps_proj(int64_t bucket, std::vector & out) { - const int64_t half = 128; const float lm = std::log(10000.0f); const float t = (float) bucket; - out.assign(256, 0.0f); - for (int64_t i=0; i & out) { - const int64_t half = dim/2; const float step = std::log(10000.0f)/(float) half; const float t = (float) bucket; - out.assign((size_t) T * dim, 0.0f); - for (int64_t tk=0; tk sinusoidal_time_emb(double t, int64_t dim, double min_p, double max_p) { - const int64_t half = dim/2; - std::vector out(dim); - for (int64_t i=0; i & out) { - out.assign((size_t) seq * seq, 0.0f); - const float NEG = -std::numeric_limits::infinity(); - for (int64_t q=0; qBF16 round trip ggml + // --act-dtype bf16, which removes the per-GEMM F32<->BF16 round trip ggml // pays when BF16 weights meet F32 activations. See mm_act/as_type below. ggml_type act_type = GGML_TYPE_F32; @@ -566,6 +566,7 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { if (!preprocess_image_chw(in.images[v], image_size, chw)) { return {}; } ggml_backend_tensor_set(t_px[v], chw.data(), 0, ggml_nbytes(t_px[v])); } + graph_unique_names(vg); if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(evo1): vision graph compute failed (%lld views)\n", (long long) n_views); return {}; @@ -799,6 +800,7 @@ std::vector Evo1ModelArch::predict(const Inputs& in) { { std::vector qm(SEQ, 0.0f); for (int64_t p=0; p Gr00tN1d5ModelArch::predict(const Inputs& in) { for (int64_t v=0; v Gr00tN1d5ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_tproj[s], tpr.data(), 0, ggml_nbytes(gio.t_tproj[s])); } + graph_unique_names(main_graph.graph()); const auto tc0 = std::chrono::steady_clock::now(); const ggml_status status = ggml_backend_graph_compute(backend, main_graph.graph()); const auto tc1 = std::chrono::steady_clock::now(); diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index 52f14fe..d2e49aa 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -374,6 +374,7 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { } if (vok) { ggml_backend_tensor_set(t_patches, patches_all.data(), 0, ggml_nbytes(t_patches)); + graph_unique_names(vgA); if (ggml_backend_graph_compute(backend, vgA) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d6): vision compute A failed\n"); vok = false; @@ -386,6 +387,7 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { shuf_host.data()+(size_t) v*c4*K); ggml_backend_tensor_set(t_shuf, shuf_host.data(), 0, ggml_nbytes(t_shuf)); + graph_unique_names(vgB); if (ggml_backend_graph_compute(backend, vgB) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d6): vision compute B failed\n"); vok = false; @@ -515,6 +517,7 @@ std::vector Gr00tN1d6ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(gio.t_tproj[s], tpr.data(), 0, ggml_nbytes(gio.t_tproj[s])); } + graph_unique_names(main_graph.graph()); const auto tc0 = std::chrono::steady_clock::now(); const ggml_status status = ggml_backend_graph_compute(backend, main_graph.graph()); const auto tc1 = std::chrono::steady_clock::now(); diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index ce19844..a64cf7e 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -414,6 +414,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_cos, rope_cos.data(), 0, ggml_nbytes(t_cos)); ggml_backend_tensor_set(t_sin, rope_sin.data(), 0, ggml_nbytes(t_sin)); ggml_backend_tensor_set(t_patches, patches.data(), 0, ggml_nbytes(t_patches)); + graph_unique_names(vg); if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(gr00tn1d7): vision compute failed\n"); vok = false; @@ -699,6 +700,7 @@ std::vector Gr00tN1d7ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_tproj[s], c_tproj[(size_t) s].data(), 0, ggml_nbytes(t_tproj[s])); } + graph_unique_names(gf); const auto tc0 = std::chrono::steady_clock::now(); const ggml_status st = ggml_backend_graph_compute(backend, gf); const auto tc1 = std::chrono::steady_clock::now(); diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index ed107bf..c355b43 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -272,6 +272,8 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { return {}; } } + // ImageNet constants as bf16 rounds them (0.485 -> 0.484375). The reference + // preprocesses in bf16, so these are the values it actually sees. static const float DMEAN[3]={0.484375f,0.455078125f,0.40625f}, DSTD[3]={0.228515625f,0.2236328125f,0.224609375f}; static const float SMEAN[3]={0.5f,0.5f,0.5f}, SSTD[3]={0.5f,0.5f,0.5f}; @@ -299,6 +301,7 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { normalize_tower(in.images[v],S,DMEAN,DSTD,dbuf); ggml_backend_tensor_set(px_d[v],dbuf.data(),0,ggml_nbytes(px_d[v])); normalize_tower(in.images[v],S,SMEAN,SSTD,sbuf); ggml_backend_tensor_set(px_s[v],sbuf.data(),0,ggml_nbytes(px_s[v])); } + graph_unique_names(vg); if(ggml_backend_graph_compute(backend,vg)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(openvla_oft): vision compute failed\n"); return {}; } ggml_backend_tensor_get(proj,proj_host.data(),0,proj_host.size()*sizeof(float)); stats.ms_vision = std::chrono::duration(clock::now()-tv).count(); @@ -415,6 +418,7 @@ std::vector OpenVlaOftModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(act0,z.data(),0,ggml_nbytes(act0)); } + graph_unique_names(gf); if(ggml_backend_graph_compute(backend,gf)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(openvla_oft): main compute failed\n"); return {}; } std::vector na((size_t)action_dim*chunk); ggml_backend_tensor_get(norm_actions,na.data(),0,na.size()*sizeof(float)); diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index c70dcca..43c8076 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -26,7 +26,7 @@ #include "gguf.h" #include "gguf_reader.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "modules/preprocess.h" #include "act_dtype.h" #include "cuda/vla_cuda_ops.h" @@ -303,28 +303,33 @@ bool load_stats(gguf_reader & g, Pi0ModelArch & m) { m.state_std .assign(cfg.real_state_dim, 1.f); m.action_mean.assign(cfg.real_action_dim, 0.f); m.action_std .assign(cfg.real_action_dim, 1.f); + // Absent stats are a valid checkpoint: identity, carry on. Stats that are + // present but unreadable are not - falling back to identity there hands back + // un-denormalised actions with nothing in the log. Note stderr, not stdout: + // stdout is the action stream tests/predict_check.cpp diffs. auto read1d = [&](const char * name, std::vector & dst) { const ggml_tensor * t = g.meta(name); if (!t) { - std::printf("vla(pi0): %s missing - identity\n", name); - return; + std::fprintf(stderr, "vla(pi0): %s missing - identity\n", name); + return true; } if (t->ne[0] != (int64_t) dst.size()) { - std::printf("vla(pi0): %s dim mismatch - identity\n", name); - return; + std::fprintf(stderr, "vla(pi0): %s is %lld wide, expected %zu\n", + name, (long long) t->ne[0], dst.size()); + return false; } - const std::vector identity = dst; if (!g.read_raw(name, dst.data(), dst.size()*sizeof(float))) { - // A short read leaves dst half-overwritten. - dst = identity; - std::printf("vla(pi0): %s read failed - identity\n", name); + std::fprintf(stderr, "vla(pi0): %s read failed\n", name); + return false; } + return true; }; - read1d("state_mean", m.state_mean); - read1d("state_std", m.state_std); - read1d("action_mean", m.action_mean); - read1d("action_std", m.action_std); - return true; + bool ok = true; + ok &= read1d("state_mean", m.state_mean); + ok &= read1d("state_std", m.state_std); + ok &= read1d("action_mean", m.action_mean); + ok &= read1d("action_std", m.action_std); + return ok; } } @@ -510,6 +515,7 @@ std::vector Pi0ModelArch::predict(const Inputs& in) { for (int v=0; v Pi0ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_time[s], tile.data(), 0, ggml_nbytes(t_time[s])); } + graph_unique_names(gf); const auto ti0 = clk::now(); const ggml_status st = ggml_backend_graph_compute(backend, gf); stats.ms_inference = std::chrono::duration(clk::now()-ti0).count(); diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index f16bc56..6a0079b 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -26,7 +26,7 @@ #include "gguf.h" #include "gguf_reader.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "modules/preprocess.h" #include "env_flag.h" @@ -337,35 +337,40 @@ bool load_stats(gguf_reader & g, Pi05ModelArch & m) { m.state_std .assign(cfg.real_state_dim, 1.f); m.action_mean.assign(cfg.real_action_dim, 0.f); m.action_std .assign(cfg.real_action_dim, 1.f); + // Absent stats are a valid checkpoint: identity, carry on. Stats that are + // present but unreadable are not - falling back to identity there hands back + // un-denormalised actions with nothing in the log. Note stderr, not stdout: + // stdout is the action stream tests/predict_check.cpp diffs. auto read1d = [&](const char * name, std::vector & dst) { const ggml_tensor * t = g.meta(name); if (!t) { - std::printf("vla(pi05): %s missing - identity\n", name); - return; + std::fprintf(stderr, "vla(pi05): %s missing - identity\n", name); + return true; } if (t->ne[0] != (int64_t) dst.size()) { - std::printf("vla(pi05): %s dim mismatch - identity\n", name); - return; + std::fprintf(stderr, "vla(pi05): %s is %lld wide, expected %zu\n", + name, (long long) t->ne[0], dst.size()); + return false; } - const std::vector identity = dst; if (!g.read_raw(name, dst.data(), dst.size()*sizeof(float))) { - // A short read leaves dst half-overwritten. - dst = identity; - std::printf("vla(pi05): %s read failed - identity\n", name); + std::fprintf(stderr, "vla(pi05): %s read failed\n", name); + return false; } + return true; }; - read1d("state_mean", m.state_mean); - read1d("state_std", m.state_std); - read1d("action_mean", m.action_mean); - read1d("action_std", m.action_std); + bool ok = true; + ok &= read1d("state_mean", m.state_mean); + ok &= read1d("state_std", m.state_std); + ok &= read1d("action_mean", m.action_mean); + ok &= read1d("action_std", m.action_std); if (m.quantile_norm) { m.action_q01.assign(cfg.real_action_dim, -1.f); m.action_q99.assign(cfg.real_action_dim, 1.f); - read1d("action_q01", m.action_q01); - read1d("action_q99", m.action_q99); + ok &= read1d("action_q01", m.action_q01); + ok &= read1d("action_q99", m.action_q99); } - return true; + return ok; } } @@ -555,6 +560,7 @@ std::vector Pi05ModelArch::predict(const Inputs& in) { for (int v=0; v Pi05ModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_time[s], tv.data(), 0, ggml_nbytes(t_time[s])); } + graph_unique_names(gf); const auto ti0 = clk::now(); const ggml_status st = ggml_backend_graph_compute(backend, gf); stats.ms_inference = std::chrono::duration(clk::now()-ti0).count(); diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index 62c062c..4e522c8 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -21,7 +21,7 @@ #include "model.h" #include "modules/preprocess.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "ggml.h" #include "ggml-backend.h" @@ -1241,9 +1241,9 @@ SmolVLAModelArch* smolvla_load_impl(ggml_type weight_dtype, cfg.expert_h, cfg.n_suffix); } - m->weight_buf = ggml_backend_alloc_ctx_tensors(m->ctx_weights, m->backend); + m->weight_buf = alloc_weights(m->ctx_weights, m->backend); if (!m->weight_buf) { - std::fprintf(stderr, "vla: ggml_backend_alloc_ctx_tensors (weights) failed\n"); + std::fprintf(stderr, "vla: alloc_weights failed\n"); delete m; return nullptr; } @@ -1551,12 +1551,14 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { break; } ggml_backend_tensor_set(t_px, chw.data(), 0, ggml_nbytes(t_px)); + graph_unique_names(gA); if (ggml_backend_graph_compute(m->backend, gA) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(smolvla): vision compute A failed (view %d)\n", v); vok = false; break; } ggml_backend_tensor_get(post_ln, post_host.data(), 0, ggml_nbytes(post_ln)); pixel_shuffle_hf(post_host.data(), shuf_host.data(), H, grid, s); ggml_backend_tensor_set(t_shuf, shuf_host.data(), 0, ggml_nbytes(t_shuf)); + graph_unique_names(gB); if (ggml_backend_graph_compute(m->backend, gB) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(smolvla): connector compute failed (view %d)\n", v); vok = false; break; } @@ -1687,6 +1689,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_backend_tensor_set(m->in_pos_full, pos_full_host.data(), 0, pos_full_host.size() * sizeof(int32_t)); ggml_backend_tensor_set(m->in_pos_rebased, pos_rebased_host.data(), 0, pos_rebased_host.size() * sizeof(int32_t)); + graph_unique_names(m->gf_cached); const auto t0 = clock::now(); if (ggml_backend_graph_compute(m->backend, m->gf_cached) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla: ggml compute (cached) failed\n"); @@ -1868,8 +1871,13 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { x_t = ggml_add(ctx, x_t, ggml_scale(ctx, v_t, dt)); } + // Graph inputs, not weights: tag them so a backend that reads buffer usage + // (ggml-openvino) does not mistake the default ANY for a KV cache. gallocr + // tags its own arena the same way. ggml_backend_buffer_t compute_buf = ggml_backend_alloc_ctx_tensors(ctx, m->backend); - if (!compute_buf) { + if (compute_buf) { + ggml_backend_buffer_set_usage(compute_buf, GGML_BACKEND_BUFFER_USAGE_COMPUTE); + } else { std::fprintf(stderr, "vla: ggml_backend_alloc_ctx_tensors (compute) failed\n"); ggml_free(ctx); return {}; @@ -1902,6 +1910,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { ggml_build_forward_expand(gf_pre, k_cache[i]); ggml_build_forward_expand(gf_pre, v_cache[i]); } + graph_unique_names(gf_pre); const auto t0 = clock::now(); if (ggml_backend_graph_compute(m->backend, gf_pre) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla: ggml prefill compute failed\n"); @@ -1920,6 +1929,7 @@ std::vector predict_impl(SmolVLAModelArch* m, const Inputs& in) { { ggml_cgraph * gf = ggml_new_graph_custom(ctx, 16384, false); ggml_build_forward_expand(gf, x_t); + graph_unique_names(gf); const auto t0 = clock::now(); if (ggml_backend_graph_compute(m->backend, gf) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla: ggml compute failed\n"); diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index af00615..12821e0 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -25,7 +25,7 @@ #include "gguf.h" #include "gguf_reader.h" #include "scratch_ctx.h" -#include "models/dit_common.h" +#include "layers/embed.h" #include "env_flag.h" #include @@ -315,6 +315,8 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { return {}; } } + // ImageNet constants as bf16 rounds them (0.485 -> 0.484375). The reference + // preprocesses in bf16, so these are the values it actually sees. static const float DMEAN[3]={0.484375f,0.455078125f,0.40625f}, DSTD[3]={0.228515625f,0.2236328125f,0.224609375f}; static const float SMEAN[3]={0.5f,0.5f,0.5f}, SSTD[3]={0.5f,0.5f,0.5f}; @@ -342,6 +344,7 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { normalize_tower(in.images[v],S,DMEAN,DSTD,dbuf); ggml_backend_tensor_set(px_d[v],dbuf.data(),0,ggml_nbytes(px_d[v])); normalize_tower(in.images[v],S,SMEAN,SSTD,sbuf); ggml_backend_tensor_set(px_s[v],sbuf.data(),0,ggml_nbytes(px_s[v])); } + graph_unique_names(vg); if(ggml_backend_graph_compute(backend,vg)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(vla_adapter): vision compute failed\n"); return {}; } ggml_backend_tensor_get(proj,proj_host.data(),0,proj_host.size()*sizeof(float)); stats.ms_vision = std::chrono::duration(clock::now()-tv).count(); @@ -500,6 +503,7 @@ std::vector VlaAdapterModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(cc,cb.data(),0,ggml_nbytes(cc)); ggml_backend_tensor_set(ss,sb.data(),0,ggml_nbytes(ss)); }; fill_cs(cT,sT,chunk); fill_cs(cA,sA,num_tokens+1); fill_cs(cK,sK,NPATCH); + graph_unique_names(gf); if(ggml_backend_graph_compute(backend,gf)!=GGML_STATUS_SUCCESS){ std::fprintf(stderr,"vla(vla_adapter): main compute failed\n"); return {}; } std::vector na((size_t)action_dim*chunk); ggml_backend_tensor_get(norm_actions,na.data(),0,na.size()*sizeof(float)); diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index 27bc6a0..d297b91 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -426,6 +426,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_pos, c_pos_interp.data(), 0, ggml_nbytes(t_pos)); ggml_backend_tensor_set(t_cos, c_rope_cos.data(), 0, ggml_nbytes(t_cos)); ggml_backend_tensor_set(t_sin, c_rope_sin.data(), 0, ggml_nbytes(t_sin)); + graph_unique_names(vg); if (ggml_backend_graph_compute(backend, vg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): vision compute failed\n"); vok = false; @@ -565,6 +566,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { ggml_backend_tensor_set(t_ds[j], ds_pad[j].data(), 0, ggml_nbytes(t_ds[j])); const auto tp0 = std::chrono::steady_clock::now(); + graph_unique_names(lg); if (ggml_backend_graph_compute(backend, lg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): LM compute failed\n"); return {}; } stats.ms_prefill = std::chrono::duration(std::chrono::steady_clock::now()-tp0).count(); if (dump_prefix) { @@ -670,6 +672,7 @@ std::vector VlaJepaModelArch::predict(const Inputs& in) { } const auto td0 = std::chrono::steady_clock::now(); + graph_unique_names(hg); if (ggml_backend_graph_compute(backend, hg) != GGML_STATUS_SUCCESS) { std::fprintf(stderr, "vla(vla_jepa): head compute failed\n"); return {}; } stats.ms_denoise = std::chrono::duration(std::chrono::steady_clock::now()-td0).count(); stats.ms_inference = stats.ms_prefill+stats.ms_denoise; diff --git a/src/scratch_ctx.h b/src/scratch_ctx.h index 8db90ae..4e6a67e 100644 --- a/src/scratch_ctx.h +++ b/src/scratch_ctx.h @@ -41,12 +41,20 @@ class scratch_ctx { } ggml_context * reset(size_t arena) { + // Growing matters: an arena sized from the input shape would otherwise + // keep the first call's smaller pool and abort in ggml_new_tensor. + // Every call site passes a constant today. + if (ctx_ && arena > arena_) { + ggml_free(ctx_); + ctx_ = nullptr; + } if (ctx_) { ggml_reset(ctx_); return ctx_; } ggml_init_params p = { arena, nullptr, true }; - ctx_ = ggml_init(p); + ctx_ = ggml_init(p); + arena_ = arena; return ctx_; } @@ -70,6 +78,7 @@ class scratch_ctx { private: ggml_context * ctx_ = nullptr; ggml_gallocr_t galloc_ = nullptr; + size_t arena_ = 0; }; // Key is whatever shape the graph depends on (it needs operator==); IO holds the diff --git a/src/vlm/engine.cpp b/src/vlm/engine.cpp index e79c147..ab956ff 100644 --- a/src/vlm/engine.cpp +++ b/src/vlm/engine.cpp @@ -135,7 +135,8 @@ bool Engine::decode_image_file(const std::string & path, Image & out) const { if (!loaded()) { return false; } - mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(impl_->vision.get(), path.c_str(), false).bitmap); + mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(impl_->vision.get(), path.c_str(), false, + mtmd_helper_init_opt_default()).bitmap); return bitmap_to_image(bmp, out); } @@ -143,7 +144,8 @@ bool Engine::decode_image_buf(const uint8_t * data, size_t len, Image & out) con if (!loaded() || !data || len == 0) { return false; } - mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(impl_->vision.get(), data, len, false).bitmap); + mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_buf(impl_->vision.get(), data, len, false, + mtmd_helper_init_opt_default()).bitmap); return bitmap_to_image(bmp, out); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5e60cc4..e3175f7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,13 @@ target_link_libraries(test_dit_common PRIVATE ggml) target_compile_options(test_dit_common PRIVATE -Wall -Wextra) add_test(NAME dit_common COMMAND test_dit_common) +# Defines GGML_USE_OPENVINO itself so the pass is exercised on any build. +add_executable(test_graph_names test_graph_names.cpp) +target_include_directories(test_graph_names PRIVATE ${CMAKE_SOURCE_DIR}/src) +target_link_libraries(test_graph_names PRIVATE ggml) +target_compile_options(test_graph_names PRIVATE -Wall -Wextra) +add_test(NAME graph_names COMMAND test_graph_names) + add_executable(test_qwen3vl_vit test_qwen3vl_vit.cpp) target_include_directories(test_qwen3vl_vit PRIVATE ${CMAKE_SOURCE_DIR}/src) target_link_libraries(test_qwen3vl_vit PRIVATE ggml) @@ -40,9 +47,14 @@ target_link_libraries(test_config_guard PRIVATE vla_core) target_compile_options(test_config_guard PRIVATE -Wall -Wextra) add_test(NAME config_guard COMMAND test_config_guard) -# The A/B harness for the two BitVLA ternary-GEMM tilings (bitvla_gemm_check.cu) -# was never committed, so there is no target for it here. VLA_BITVLA_NARROW_GEMM=1 -# still selects the old one-tile-per-CTA kernel for a hand-run comparison. +# A/B harness for the two BitVLA ternary-GEMM tilings. Built so it cannot rot, +# not registered with ctest: it needs a GPU and is read by hand. +# VLA_BITVLA_NARROW_GEMM=1 selects the old one-tile-per-CTA kernel at runtime. +if(GGML_CUDA) + add_executable(bitvla_gemm_check bitvla_gemm_check.cu) + target_include_directories(bitvla_gemm_check PRIVATE ${CMAKE_SOURCE_DIR}/src) + target_link_libraries(bitvla_gemm_check PRIVATE bitvla_cuda_kernels) +endif() # Regression test for the in-tree BF16 CUDA kernels and the ggml hook they ride # on. Skips itself (exit 0) when no CUDA device is present. diff --git a/tests/bitvla_gemm_check.cu b/tests/bitvla_gemm_check.cu index 74a73b7..745af5d 100644 --- a/tests/bitvla_gemm_check.cu +++ b/tests/bitvla_gemm_check.cu @@ -26,7 +26,7 @@ * the LM and ViT actually use, plus a ragged M to exercise the row tail. */ -#include "../src/kernels/bitvla/bitnet_kernels.h" +#include "kernels/bitvla/bitnet_kernels.h" #include #include diff --git a/tests/test_dit_common.cpp b/tests/test_dit_common.cpp index 6387f37..1697538 100644 --- a/tests/test_dit_common.cpp +++ b/tests/test_dit_common.cpp @@ -17,7 +17,7 @@ // end-to-end sha oracle cannot do for three of the four archs. The trig orders // are opposite and both match the reference. -#include "models/dit_common.h" +#include "layers/embed.h" #undef NDEBUG // keep assert() live even in Release builds #include diff --git a/tests/test_graph_names.cpp b/tests/test_graph_names.cpp new file mode 100644 index 0000000..0ee8d4f --- /dev/null +++ b/tests/test_graph_names.cpp @@ -0,0 +1,77 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Pins vla::graph_unique_names, which ggml-openvino depends on to tell two +// same-named nodes apart. Defines GGML_USE_OPENVINO so the body compiles on any +// build; nothing here calls into the backend, so only ggml is linked. +// +// The bug this exists to catch: renaming through ggml_format_name passes the +// tensor's own name as the "%s" source, and glibc empties it instead of +// appending, so every duplicate collapsed to "#". + +#define GGML_USE_OPENVINO +#include "backend.h" + +#undef NDEBUG // keep assert() live even in Release builds +#include +#include +#include +#include +#include +#include + +int main() { + ggml_init_params p = { 64u * 1024 * 1024, nullptr, true }; + ggml_context * C = ggml_init(p); + assert(C); + + ggml_tensor * x = ggml_new_tensor_2d(C, GGML_TYPE_F32, 8, 4); + ggml_set_name(x, "x"); + + // Unnamed reshapes all land on the same ggml-derived name. + ggml_cgraph * gf = ggml_new_graph_custom(C, 256, false); + for (int i = 0; i < 6; ++i) + ggml_build_forward_expand(gf, ggml_reshape_2d(C, ggml_scale(C, x, 1.0f + i), 4, 8)); + + const int n = ggml_graph_n_nodes(gf); + assert(n >= 12); + + std::vector before; + for (int i = 0; i < n; ++i) + before.emplace_back(ggml_get_name(ggml_graph_node(gf, i))); + + // Precondition: without the pass the graph really does carry duplicates. + assert(std::set(before.begin(), before.end()).size() < (size_t) n); + + vla::graph_unique_names(gf); + + std::set post; + for (int i = 0; i < n; ++i) { + const std::string nm = ggml_get_name(ggml_graph_node(gf, i)); + assert(!nm.empty()); + assert(post.insert(nm).second); // every node distinct + assert(nm.find(before[i]) != std::string::npos); // and still says what it was + } + + // Idempotent: the graph cache hands the same graph back on every predict. + std::set again; + vla::graph_unique_names(gf); + for (int i = 0; i < n; ++i) + again.insert(ggml_get_name(ggml_graph_node(gf, i))); + assert(again == post); + + ggml_free(C); + std::printf("test_graph_names: OK (%d nodes)\n", n); + return 0; +}