Support backend OpenVINO

.github/workflows/build.yml

@@ -16,7 +16,7 @@ jobs:
   cpp-unit:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       # Compiled directly: pure, no llama.cpp or protobuf/zmq needed.
       - name: pure unit tests
         run: |
@@ -29,8 +29,8 @@ jobs:
   py-tooling:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
         with:
           python-version: '3.11'
       - name: converter remap
@@ -41,21 +41,36 @@ jobs:
   build-gate:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - name: deps
         run: |
           sudo apt-get update -qq
           sudo apt-get install -y -qq --no-install-recommends \
             build-essential cmake git ca-certificates pkg-config \
             libzmq3-dev cppzmq-dev libprotobuf-dev protobuf-compiler
-      - uses: actions/cache@v4
+      # Read the pin rather than repeat it: two copies drift and the stale one
+      # silently reuses the wrong _deps.
+      - name: read llama.cpp pin
+        id: pin
+        run: echo "tag=$(bash scripts/llama_tag.sh)" >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@v6
         with:
           path: build/_deps
-          key: llama-b10331-${{ runner.os }}
+          key: llama-${{ steps.pin.outputs.tag }}-${{ runner.os }}
       # Everything, not a target list: ctest registers tests this job must build,
       # and a named list goes stale the next time one is added.
       - name: build + ctest (CPU, -Wall -Wextra)
         run: |
           cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=OFF -DVLA_BUILD_TESTS=ON
           cmake --build build -j"$(nproc)"
           ctest --test-dir build --output-on-failure
+      # This job is the only one that fetches llama.cpp, and neither patch script
+      # runs on a CPU build, so their anchors would otherwise rot unnoticed until
+      # someone configures a CUDA or OpenVINO tree. Patch a copy: the real one is
+      # cached.
+      - name: patch anchors still apply
+        run: |
+          cp -r build/_deps/llama-src /tmp/llama-patchtest
+          python3 scripts/patch_ggml_cuda_ext_hook.py /tmp/llama-patchtest
+          python3 scripts/patch_ggml_openvino.py /tmp/llama-patchtest
+          python3 scripts/patch_ggml_openvino.py /tmp/llama-patchtest   # idempotent

.github/workflows/release.yml

@@ -41,7 +41,7 @@ jobs:
             cuda: false
             runner: ubuntu-24.04-arm
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
       - name: deps
         run: |
@@ -75,15 +75,15 @@ jobs:
           mkdir -p "$out/scripts" && cp scripts/tokenize_prompt.py "$out/scripts/"
           tar -czf "$out.tar.gz" "$out"
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         with:
           name: ${{ matrix.name }}
           path: '*.tar.gz'
 
   macos:
     runs-on: macos-14
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
 
       - name: deps
         run: brew install cmake zeromq cppzmq protobuf
@@ -105,25 +105,25 @@ jobs:
           find build -name 'default.metallib' -exec cp {} "$out/" \;
           tar -czf "$out.tar.gz" "$out"
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         with:
           name: macos-arm64-metal
           path: '*.tar.gz'
 
   docker:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v4
-      - uses: docker/setup-buildx-action@v3
-      - uses: docker/login-action@v3
+      - uses: actions/checkout@v7
+      - uses: docker/setup-buildx-action@v4
+      - uses: docker/login-action@v4
         if: startsWith(github.ref, 'refs/tags/')
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: image name
         run: echo "IMAGE=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV"
-      - uses: docker/build-push-action@v6
+      - uses: docker/build-push-action@v7
         with:
           context: .
           push: ${{ startsWith(github.ref, 'refs/tags/') }}
@@ -140,9 +140,9 @@ jobs:
     steps:
       # Tarballs only. The docker job also leaves a .dockerbuild build record
       # artifact behind, and pulling that one fails the whole download.
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v8
         with: { path: dist, pattern: '{linux,macos}-*', merge-multiple: true }
-      - uses: softprops/action-gh-release@v2
+      - uses: softprops/action-gh-release@v3
         with:
           files: dist/*.tar.gz
           fail_on_unmatched_files: true

.github/workflows/vla-ci.yml

@@ -23,10 +23,10 @@ jobs:
     runs-on: [self-hosted, vla-ci-orchestrator]
     timeout-minutes: 240
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
       - name: Sweep all platforms (parallel) + gate
         run: bash ci/orchestrate.sh all
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v7
         if: always()
         with:
           name: ci-results

.gitignore

@@ -1,6 +1,5 @@
 # Build
-build/
-build-*/
+build*/
 out/
 cmake-build-*/
 _workdir*
@@ -48,7 +47,6 @@ outputs/
 weights/
 CLAUDE.md
 
-third_party/llama.cpp/
 eval/sim/libero/LIBERO/
 eval/sim/libero/libero_uv/
 eval/sim/simpler/SimplerEnv/

CHANGELOG.md

@@ -2,6 +2,113 @@
 
 Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://keepachangelog.com).
 
+## [Unreleased]
+
+### Added
+
+- **OpenVINO backend.** `-DGGML_OPENVINO=ON` runs the archs on Intel CPUs, iGPUs
+  and NPUs through ggml's OpenVINO backend. SmolVLA, π0.5, Evo-1 and VLA-Adapter
+  match an F32 CPU reference to 1e-3; on an Arc B390 iGPU that is 3.0x to 9.6x
+  the native CPU backend. All nine tested archs are inside the accuracy bar on the
+  OpenVINO CPU plugin and on the iGPU. OpenVLA-OFT is untested.
+  See `docs/backend/ov.md`.
+- `scripts/install_ov.sh` installs the OpenVINO runtime and the Intel GPU/NPU
+  driver stack on Ubuntu 22.04 and 24.04, with the runtime archive checksummed
+  against a digest pinned in the script.
+- `scripts/patch_ggml_openvino.py` applies thirteen fixes to the fetched
+  ggml-openvino sources at configure time. Each hunk is checked on its own, so a
+  `build/_deps` patched by an older checkout fails loudly instead of building
+  something quietly wrong.
+- `tests/test_graph_names.cpp` pins `vla::graph_unique_names`.
+- CI now checks that both llama.cpp patch scripts still apply, on a copy of
+  the fetched tree. Neither ran on a CPU build, so their anchors could rot
+  unnoticed until someone configured a CUDA or OpenVINO tree.
+- `docs/UPSTREAMING.md` and `scripts/upstream_split.py` regroup the thirteen
+  ggml-openvino fixes into one llama.cpp branch per PR. They are generic
+  backend defects, not vla.cpp workarounds; landing them upstream removes the
+  configure-time patch step entirely.
+
+### Fixed
+
+- Two elementwise adds stacked on a GEMM came out wrong on the Intel iGPU. The
+  GPU plugin folds elementwise ops into the preceding GEMM as post-ops, and given
+  `ADD(ADD(residual, GEMM), graph_input)` it folds both and silently drops the
+  second operand - the result equals the inner add. A llama.cpp graph never builds
+  that chain; a VLA does, wherever a vision tower's features are added on top of an
+  FFN residual. VLA-JEPA (5.4e-1) and GR00T N1.7 (1.9e0) were wrong on the iGPU
+  while matching the CPU plugin to 1e-4. Re-associating the two adds so the GEMM
+  keeps one post-op puts both at 2.6e-3. Bisected with `GGML_OPENVINO_DEBUG_NODE`.
+- π0's action dims drifted 4e-2 on the iGPU and its gripper flipped a step late,
+  because the GPU plugin computes in F16 and π0 unrolls its whole denoise loop
+  inside one graph. `GGML_OPENVINO_GPU_PRECISION` now exposes the plugin's
+  inference precision; `backend_init` defaults it to f32 for π0 alone, which costs
+  about 3x on that arch and puts it at 6.5e-5.
+- `scripts/patch_ggml_openvino.py` now fails if `EDITS` has a duplicate key. Python
+  keeps the last one silently, and a duplicate briefly removed the whole Intel
+  OpenCL platform fix from the patch without any error.
+- The position-input fix stopped running when llama.cpp moved to `b10729`. That
+  release relocated the naming out of `GgmlOvDecoder::get_graph_input_ov_name()`,
+  which the patch guards, into a new free `get_tensor_graph_input_ov_name()`, and
+  left the member behind with no callers. The hunk still applied cleanly, so
+  nothing failed loudly - SmolVLA and π0.5 simply stopped returning actions
+  ("Argument shapes are inconsistent", a 113-token prefix ROPE reading the
+  50-token suffix's table). Both functions are guarded now, and the patch script
+  says to check for a live caller, not just a matching anchor, on every tag bump.
+- `scripts/upstream_split.py` addressed hunks by position in the patch script's
+  edit list. Adding a hunk to the front of a file's list silently handed every
+  later hunk to the wrong branch, and its own coverage count still read 29/29
+  because each index was still used exactly once. Two branches had been swapped
+  this way. Hunks are now addressed by a unique substring of their anchor, which
+  fails loudly instead. The PERMUTE `op_case` fix, which had no branch at all,
+  now has one.
+- `graph_unique_names` renamed through `ggml_format_name`, which passes the
+  tensor's own name to `vsnprintf` as both destination and `%s` source. glibc
+  empties it, so every duplicate node became the bare string `#<index>`.
+- The OpenVINO naive-path compiled-model cache was keyed on node count plus the
+  first and last node name. Two graphs of the same size collided and the second
+  ran the first's compiled model. It now also keys on every node's op and shape,
+  and the map is bounded.
+- `GGML_OPENVINO_NAIVE_GRAPH_SIZE` went through `atoi`, so junk parsed to 0 and
+  sent every graph down the decoder-only-LLM path with nothing said. Empty
+  environment values no longer count as a setting either.
+- `GGML_OPENVINO_CACHE_DIR` is cleared rather than warned about: a warm cache
+  returns wrong actions, and stderr is not always read. `VLA_ALLOW_OV_CACHE=1`
+  keeps it.
+- `scripts/print_versions.sh` printed `?` for the llama.cpp pin ever since the
+  tag moved behind `VLA_LLAMA_TAG`.
+- The OpenVINO `find_package` failure message was unreachable, sitting after the
+  fetch whose own `find_package(REQUIRED)` fired first.
+- BitVLA indexed its action slots as `seq-2-n_action+i` with no check that the
+  sequence is long enough. Neither `ggml_get_rows` nor the CUDA gather
+  bound-checks, so a short prompt read out of bounds and returned it as hidden
+  states. One guard now covers both LM paths.
+- pi0 and pi0.5 fell back to identity normalisation stats on a dimension mismatch
+  or a short read, and said so on stdout. That returns un-denormalised actions
+  from a checkpoint that looked fine. Both now fail the load, and the message
+  goes to stderr - stdout is the action stream `predict_check` diffs.
+- `scratch_ctx::reset` ignored an arena larger than the first call's, which would
+  abort in `ggml_new_tensor` if any call site ever sized one from the input.
+- The safetensors arch probe would allocate up to 256 MB for a header it only
+  substring-searches. Capped at 16 MB.
+- The two CUDA targets were the only first-party code built without
+  `-Wall -Wextra`.
+- `tests/bitvla_gemm_check.cu` had no build target and a comment claiming it was
+  never committed. It builds now, under `GGML_CUDA`.
+- Stale references to `vision_common.h` (now `modules/preprocess.h`) and to the
+  retired `VLA_EVO1_BF16_ACT` switch.
+
+### Changed
+
+- llama.cpp pinned at `b10729`, up from `b10331`. Brings OpenVINO 2026.3.1, the
+  IM2COL+MatMul to native-convolution fusion, and the `RELU`/`NEG`/`SQR`
+  translators, which the local patch no longer has to add. Byte-identical on the
+  CPU backend for all eleven archs. The build.yml cache key now reads the tag out
+  of `CMakeLists.txt` instead of repeating it.
+- `src/models/dit_common.h` is gone. It redefined six `vla::` functions that
+  `src/layers/` already had, with both copies linked into `vla_core`. Every
+  includer used only `sinusoidal_time_emb` or `build_causal_mask`, so they now
+  include `layers/embed.h`. Byte-identical across all 11 archs.
+
 ## [0.3.0] - 2026-08-14
 
 Every architecture is byte-identical to 0.2.0 at matching settings.

CMakeLists.txt

@@ -12,7 +12,7 @@ if(NOT CMAKE_BUILD_TYPE)
 endif()
 
 set(_vla_accel "")
-foreach(_flag GGML_CUDA GGML_SYCL GGML_METAL)
+foreach(_flag GGML_CUDA GGML_SYCL GGML_METAL GGML_OPENVINO)
     if(${_flag})
         list(APPEND _vla_accel ${_flag})
     endif()
@@ -24,6 +24,14 @@ if(_vla_accel_n GREATER 1)
         "Enable one accelerator backend at a time; got ${_vla_accel_str}. "
         "Configure a separate build directory per backend.")
 endif()
+# backend.h calls ggml_backend_<accel>_init directly, and GGML_BACKEND_DL builds
+# the accelerators as loadable modules that are not linked into ggml. Say so here
+# rather than at the undefined symbol.
+if(_vla_accel_n GREATER 0 AND GGML_BACKEND_DL)
+    message(FATAL_ERROR
+        "GGML_BACKEND_DL=ON is not supported with ${_vla_accel}: the backend is "
+        "built as a module and src/backend.h links its init directly.")
+endif()
 
 set(LLAMA_BUILD_COMMON   ON  CACHE BOOL "" FORCE)
 set(LLAMA_BUILD_TOOLS    ON  CACHE BOOL "" FORCE)
@@ -35,11 +43,36 @@ if(GGML_CUDA)
     find_package(Python3 COMPONENTS Interpreter REQUIRED)
     set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE}
         ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_cuda_ext_hook.py <SOURCE_DIR>)
+elseif(GGML_OPENVINO)
+    find_package(Python3 COMPONENTS Interpreter REQUIRED)
+    set(_vla_llama_patch PATCH_COMMAND ${Python3_EXECUTABLE}
+        ${CMAKE_CURRENT_SOURCE_DIR}/scripts/patch_ggml_openvino.py <SOURCE_DIR>)
+
+    # Before the fetch: ggml's own ggml-openvino/CMakeLists.txt does
+    # find_package(OpenVINO REQUIRED ...) and would get there first, so this
+    # message only reaches anyone if it runs earlier. Same components it asks for.
+    find_package(OpenVINO QUIET COMPONENTS Runtime Threading)
+    if(NOT OpenVINO_FOUND)
+        message(FATAL_ERROR
+            "GGML_OPENVINO=ON but the OpenVINO runtime was not found. Install it "
+            "(scripts/install_ov.sh) and 'source /opt/intel/openvino/setupvars.sh' "
+            "in the shell that configures. See docs/backend/ov.md.")
+    endif()
 endif()
 # Overridable so a regression can be bisected against another tag in a separate
 # build dir (-DVLA_LLAMA_TAG=b10326) without editing this file. The patch
-# anchors in scripts/patch_ggml_cuda_ext_hook.py are checked against the default.
-set(VLA_LLAMA_TAG "b10331" CACHE STRING "llama.cpp tag to fetch")
+# anchors in scripts/patch_ggml_cuda_ext_hook.py and scripts/patch_ggml_openvino.py
+# are checked against the default.
+set(_vla_llama_tag_default "b10729")
+set(VLA_LLAMA_TAG "${_vla_llama_tag_default}" CACHE STRING "llama.cpp tag to fetch")
+# A cache entry survives an edit to the line above, so an existing build dir keeps
+# the tag it was first configured with and quietly builds the wrong llama.cpp.
+if(NOT VLA_LLAMA_TAG STREQUAL _vla_llama_tag_default)
+    message(WARNING
+        "VLA_LLAMA_TAG is ${VLA_LLAMA_TAG}, not the default ${_vla_llama_tag_default}. "
+        "If that was not deliberate, this build dir predates the bump: reconfigure with "
+        "-DVLA_LLAMA_TAG=${_vla_llama_tag_default} or use a fresh one.")
+endif()
 
 include(FetchContent)
 FetchContent_Declare(llama
@@ -173,6 +206,12 @@ if(GGML_METAL AND NOT GGML_CUDA AND NOT GGML_SYCL)
     target_compile_definitions(vla_core PUBLIC GGML_USE_METAL)
 endif()
 
+if(GGML_OPENVINO AND NOT GGML_CUDA AND NOT GGML_SYCL AND NOT GGML_METAL)
+    # Only tells the archs which branch of the backend.h ladder to compile; the
+    # toolkit was located before the fetch above.
+    target_compile_definitions(vla_core PUBLIC GGML_USE_OPENVINO)
+endif()
+
 add_library(vlm_core
     src/vlm/engine.cpp
 )
@@ -286,6 +325,10 @@ target_link_libraries(vla-bench PRIVATE vla_core)
 
 # Warnings and LTO for our own targets only, never the llama.cpp subtree.
 set(VLA_FIRST_PARTY_TARGETS vla_core vlm_core vla vla-server vlm-server vla-cli vla-bench)
+# The CUDA targets were the only first-party code compiled without warnings.
+if(GGML_CUDA)
+    list(APPEND VLA_FIRST_PARTY_TARGETS bitvla_cuda_kernels vla_cuda_ops)
+endif()
 
 foreach(tgt IN LISTS VLA_FIRST_PARTY_TARGETS)
     target_compile_options(${tgt} PRIVATE $<$<COMPILE_LANGUAGE:CXX>:-Wall -Wextra>)

CONTRIBUTING.md

@@ -54,10 +54,11 @@ ckpt) model, `bitvla` for a vision-baked one.
 6. `CMakeLists.txt` - add `src/models/<name>.cpp` to `vla_core`.
 
 Then write `src/models/<name>.cpp`. Before adding a helper, check
-`src/models/`: `gguf_reader.h` (tensor and KV reads), `vision_common.h`
+`src/models/`: `gguf_reader.h` (tensor and KV reads), `modules/preprocess.h`
 (preprocessing, pixel shuffle), `dual_tower.h` (DINOv2 + SigLIP),
-`qwen3vl_vit.h` (Qwen3-VL tower), `dit_common.h` (DiT time embeddings),
-`scratch_ctx.h` (compute context reuse), `backend.h` (accelerator selection).
+`qwen3vl_vit.h` (Qwen3-VL tower), `layers/embed.h` (time embeddings, causal
+mask), `scratch_ctx.h` (compute context reuse), `backend.h` (accelerator
+selection).
 
 Your loader must fail rather than return a half-built model: check every tensor
 lookup, and check `real_*_dim <= max_*_dim` (`config_is_sane` in `src/model.cpp`