diff --git a/docs/runtime/API.md b/docs/runtime/API.md index fb29c98b..f49a0b58 100644 --- a/docs/runtime/API.md +++ b/docs/runtime/API.md @@ -1,6 +1,6 @@ # Omega Runtime API v0 -Omega Runtime API version: 0.9.0 +Omega Runtime API version: 0.13.0 Status: first written version, 2026-10-19 (slice R1, omega-mv#594, tracking #593). Source of truth is the C headers in `src/`. This file describes them; `docs/runtime/omega_runtime_api.manifest` lists them; `make test-runtime-api` checks the @@ -36,8 +36,12 @@ signature against the headers on every run. ## 2. Version and compatibility rule -The version is MAJOR.MINOR.PATCH (semantic versioning), currently **0.9.0**. It is a property of the list of entries and +The version is MAJOR.MINOR.PATCH (semantic versioning), currently **0.13.0**. It is a property of the list of entries and their behaviour, not of the library build. Changes so far: +- 0.13.0: adds experimental `omega_gpu_prefill_attn256_scoped` in `omega_gpu_attention_mirror.h`. Mode 0 preserves ordinary attention; mode 1 copies the complete original 27B K/V into one GPU-private reusable buffer before attention; mode 2 uses the provisional Omega-selected 1024-row threshold. Omega checks geometry, bounds, allocation capacity and completed PREP/copy/ATTEND ordering. Other modes and unsupported mirrored shapes are refused. `copy_ns` reports copy chip time separately from the unchanged attention info layout; total call time includes allocation and copying. Existing host/device/recorded entries and defaults are unchanged. See `om/attention-mirror/README.md` for qualification status. +- 0.12.0: adds experimental `omega_gpu_pf_qgemm_reading_option` with `(rows, index)`. The checked Omega policy returns the scoped rt, warps, wait and group values at indices 0..3, or UINT32_MAX for invalid input. This proposed adaptive profile uses 64 rows for requests at most 64 rows and 128 above that threshold. Callers obtain the checked height and use `omega_gpu_pf_qgemm_workspace_fits` to check existing row, padded-row and tile capacities before switching profiles. The query returns 1 only when all capacities cover the request; otherwise callers allocate a compatible workspace. +- 0.11.0: adds experimental `omega_gpu_pf_qgemm_scoped` per-call options and the checked `omega_gpu_pf_qgemm_scoped_bm` tile-height query. Explicit calls neither read nor modify process-global tuning knobs. Tile storage must use the queried height; actual projection dimensions are validated on every call. +- 0.10.0: adds the experimental `omega_gpu_pf_qgemm_set_group` selector. Existing entry signatures and the default execution order are unchanged. - 0.9.0 (AmperePort, CUDApp#170): grows STABLE `OmegaGpuLaunch` with `target_sm` (a MAJOR-class change taken as MINOR under the major-version-zero rule); adds EXPERIMENTAL `omega_gpu_session_target_sm`. Zero-initialized legacy launches still mean sm_121, not auto-detect. @@ -593,7 +597,7 @@ values, poison patterns) are in the INTERNAL list. | `src/omega_gpu_prefill_attn_api.h` | `OmegaGpuPrefillAttnInfo`, `OmegaGpuPrefillAttnMirrorView`, `omega_gpu_prefill_attn`, `omega_gpu_prefill_attn_release`, `omega_gpu_prefill_attn_dev`, `omega_gpu_prefill_attn_set_warps`, `omega_gpu_prefill_attn_set_fused`, `omega_gpu_prefill_attn_set_kv_mirror`, `omega_gpu_prefill_attn_kv_mirror_view`, `omega_gpu_prefill_attn_kv_mirror_invalidate`, `omega_gpu_prefill_attn_kv_mirror_forget`, `omega_gpu_prefill_attn_kv_mirror_available` | Causal prefill attention for Qwen3-30B-A3B, with an optional KV mirror; one model. | | `src/omega_gpu_moe_api.h` | `omega_gpu_moe_tensor_upload_bf16`, `omega_gpu_moe_tensor_upload_f32`, `omega_gpu_moe_tensor_free`, `omega_gpu_moe_tensor_shape`, `omega_gpu_matmul_resident_window_f32`, `omega_gpu_matmul_resident_window_bf16`, `omega_gpu_matmul_resident_windows_f32`, `omega_gpu_moe_reserve`, `omega_gpu_moe_prepare`, `omega_gpu_moe_seal`, `omega_gpu_moe_unreserve`, `omega_gpu_moe_cache_clear`, `omega_gpu_moe_set_cta_budget`, `omega_gpu_moe_cta_budget`, `omega_gpu_moe_window_byte_offset` | Column windows over a resident bf16 expert tensor (OWN-30B); tied to the serving pool. | | `src/omega_gpu_decode_graph.h` | `OmegaGpuDecodeGraphConfig`, `OmegaGpuDecodeGraphStats`, `OmegaGpuDecodeGraphVas`, `omega_gpu_decode_graph_create`, `omega_gpu_decode_graph_free`, `omega_gpu_decode_graph_set_layer`, `omega_gpu_decode_graph_set_final_norm`, `omega_gpu_decode_graph_token`, `omega_gpu_decode_graph_begin_token`, `omega_gpu_decode_graph_record_layer`, `omega_gpu_decode_graph_record_final`, `omega_gpu_decode_graph_end_token`, `omega_gpu_decode_graph_abort_token`, `omega_gpu_decode_graph_record_res`, `omega_gpu_decode_graph_record_swiglu`, `omega_gpu_decode_graph_list`, `omega_gpu_decode_graph_vas`, `omega_gpu_decode_graph_read`, `omega_gpu_decode_graph_set_stamps`, `omega_gpu_decode_graph_set_barrier`, `omega_gpu_decode_graph_barrier`, `omega_gpu_decode_graph_stamps`, `omega_gpu_decode_graph_kind_name`, `omega_gpu_decode_graph_set_flush_each`, `omega_gpu_decode_graph_set_prefetch`, `omega_gpu_decode_graph_prefetch` | One Qwen3-30B-A3B decode token as one command list; scheduling, barrier and prefetch modes are still measured. | -| `src/omega_gpu_prefill.h` | `OmegaPfWeights`, `omega_gpu_pf_rc_name`, `omega_gpu_pf_alloc_dev`, `omega_gpu_pf_alloc_host`, `omega_gpu_pf_free`, `omega_gpu_pf_upload`, `omega_gpu_pf_download`, `omega_gpu_pf_copy_va`, `omega_gpu_pf_stats`, `omega_gpu_pf_stats_reset`, `omega_gpu_pf_weights_upload`, `omega_gpu_pf_weights_free`, `omega_gpu_pf_weights_host_form`, `omega_gpu_pf_gemm_bm`, `omega_gpu_pf_gemm_tiles`, `omega_gpu_pf_gemm`, `omega_gpu_pf_gemm_set_shape`, `omega_gpu_pf_quant`, `omega_gpu_pf_qgemm_bm`, `omega_gpu_pf_qgemm_tiles`, `omega_gpu_pf_qgemm_tiles_va`, `omega_gpu_pf_qgemm`, `omega_gpu_pf_qgemm_set_shape`, `omega_gpu_pf_quant_record`, `omega_gpu_pf_qgemm_record` | Device-resident one-pass prefill for OWN-30B; arguments carry raw device addresses. | +| `src/omega_gpu_prefill.h` | `OmegaPfWeights`, `omega_gpu_pf_rc_name`, `omega_gpu_pf_alloc_dev`, `omega_gpu_pf_alloc_host`, `omega_gpu_pf_free`, `omega_gpu_pf_upload`, `omega_gpu_pf_download`, `omega_gpu_pf_copy_va`, `omega_gpu_pf_stats`, `omega_gpu_pf_stats_reset`, `omega_gpu_pf_weights_upload`, `omega_gpu_pf_weights_free`, `omega_gpu_pf_weights_host_form`, `omega_gpu_pf_gemm_bm`, `omega_gpu_pf_gemm_tiles`, `omega_gpu_pf_gemm`, `omega_gpu_pf_gemm_set_shape`, `omega_gpu_pf_quant`, `omega_gpu_pf_qgemm_bm`, `omega_gpu_pf_qgemm_tiles`, `omega_gpu_pf_qgemm_tiles_va`, `omega_gpu_pf_qgemm`, `omega_gpu_pf_qgemm_set_shape`, `omega_gpu_pf_qgemm_set_group`, `omega_gpu_pf_quant_record`, `omega_gpu_pf_qgemm_record` | Device-resident one-pass prefill for OWN-30B; arguments carry raw device addresses. | | `src/omega_gpu_pfhost.h` | `omega_gpu_pf_router_rows`, `omega_gpu_pf_router_packed_bytes`, `omega_gpu_pf_router_pack`, `omega_gpu_pf_router_logits`, `omega_gpu_pf_moe_combine`, `omega_gpu_pf_router_set_rows` | Whole-prompt prefill router logits and MoE combine for OWN-30B; arguments carry raw device addresses. | ### 9.8 General matrix multiply (`src/omega_gpu_gemm_ex.h`, added in 0.8.0) diff --git a/docs/runtime/PORTABILITY.md b/docs/runtime/PORTABILITY.md index 707a1aa8..4962bd41 100644 --- a/docs/runtime/PORTABILITY.md +++ b/docs/runtime/PORTABILITY.md @@ -23,7 +23,7 @@ Porting Omega means answering two questions that are easy to mix up: 2. **Operating system and driver.** Does this file need Linux, the NVIDIA open driver, or a C library? The table `docs/runtime/portability_table.tsv` answers both, one row per `src/omega_gpu_*`, `src/omega_blackwell_*`, -`src/omega_bw_*` and `src/omega_ampere_*` file (136 rows; the M17/M18 gate suite `omega_blackwell_gates.{c,h}` stays in Omega, which runs it against this library): +`src/omega_bw_*` and `src/omega_ampere_*` file. The table is the current inventory; the M17/M18 gate suite `omega_blackwell_gates.{c,h}` stays in Omega, which runs it against this library: | Column | Values | Meaning | |---|---|---| @@ -38,7 +38,7 @@ carries (#120, #121) brought (15 `NO_HOSTED_INCLUDE`, 6 `HOSTED_LIBC`). The sync r2 #972) adds 5 `CHIP_SPECIFIC` rows (`src/omega_bw_hazard.{c,h}`, omega-mv#554; `src/omega_gpu_fp8_slots.{c,h}` and `src/omega_gpu_fp8_internal.h`, #961) and carries omega's review of the public headers (#954): `omega_gpu_matmul_api.h`, `omega_gpu_moe_api.h`, `omega_gpu_platform.h`, `omega_gpu_serving.{c,h}` and `omega_gpu_session.h` become `CHIP_SPECIFIC`, -`omega_gpu_scan_ref.c` becomes `PORTABLE` (section 3). Totals now: 120 `CHIP_SPECIFIC`, 11 `PORTABLE`; 13 +`omega_gpu_scan_ref.c` becomes `PORTABLE` (section 3). At that sync, the totals were 120 `CHIP_SPECIFIC`, 11 `PORTABLE`; 13 `FREESTANDING_PROVEN`, 98 `NO_HOSTED_INCLUDE`, 17 `HOSTED_LIBC`, 3 `LINUX_DRIVER`. Command to recount: section 7. `make test-runtime-portability` (host only, no compiler, no GPU) checks the table against the tree: every file has a row, diff --git a/docs/runtime/omega_runtime_api.internal b/docs/runtime/omega_runtime_api.internal index 96e0a62f..daf5f68a 100644 --- a/docs/runtime/omega_runtime_api.internal +++ b/docs/runtime/omega_runtime_api.internal @@ -199,3 +199,6 @@ const OMEGA_GPU_ATTN_GMUT_TAIL_DIMS # TEST ONLY: general attention mutant ids fo const OMEGA_GPU_ATTN_GMUT_CAUSAL # TEST ONLY: general attention mutant ids for the sweep const OMEGA_GPU_ATTN_GMUT_SCALE # TEST ONLY: general attention mutant ids for the sweep const OMEGA_GPU_ATTN_GMUT_COUNT # TEST ONLY: general attention mutant ids for the sweep + +fn omega_gpu_pf_alloc_dev_locked # internal device-private allocation transport; caller already holds session lock +fn omega_gpu_pf_copy_va_locked # internal copy transport and chip-time result; caller already holds session lock diff --git a/docs/runtime/omega_runtime_api.manifest b/docs/runtime/omega_runtime_api.manifest index a91cd5e3..dd9110f6 100644 --- a/docs/runtime/omega_runtime_api.manifest +++ b/docs/runtime/omega_runtime_api.manifest @@ -12,7 +12,7 @@ # After a deliberate header change: edit the line here, run `tools/runtime_api_check.sh --bless` to refresh numbers, review the diff, # and bump the version (MAJOR/MINOR/PATCH, rule in API.md section 2; tools/runtime_api_semver.sh checks the bump against the base). # Names a manifest header declares that are NOT API go in omega_runtime_api.internal, each with a reason (the check fails otherwise). -abi omega_runtime 0.9.0 +abi omega_runtime 0.13.0 header src/omega_runtime_api.h header src/omega_gpu_platform.h header src/omega_gpu_session.h @@ -31,6 +31,7 @@ header src/omega_gpu_router_api.h header src/omega_gpu_tern_api.h header src/omega_gpu_tcgemm_api.h header src/omega_gpu_prefill_attn_api.h +header src/omega_gpu_attention_mirror.h header src/omega_gpu_moe_api.h header src/omega_gpu_decode_graph.h header src/omega_gpu_prefill.h @@ -38,9 +39,9 @@ header src/omega_gpu_pfhost.h header src/omega_gpu_gemm_ex.h # ---- version ---- const OMEGA_RUNTIME_API_VERSION_MAJOR 0 -const OMEGA_RUNTIME_API_VERSION_MINOR 9 +const OMEGA_RUNTIME_API_VERSION_MINOR 13 const OMEGA_RUNTIME_API_VERSION_PATCH 0 -const OMEGA_RUNTIME_API_VERSION 9 +const OMEGA_RUNTIME_API_VERSION 13 # ---- platform contract (src/omega_gpu_platform.h, ABI 1.2) ---- struct OmegaGpuDeviceHandle 8 4 # STABLE field OmegaGpuDeviceHandle index 0 4 u32 1 @@ -953,7 +954,10 @@ fn omega_gpu_pf_qgemm_bm fn() -> u32 # EXPERIMENTAL fn omega_gpu_pf_qgemm_tiles fn(*const OmegaGpuFp8Tensor, *const u32, *mut c_void, u32) -> u32 # EXPERIMENTAL fn omega_gpu_pf_qgemm_tiles_va fn(u32, *const u64, *const u64, *const u32, *mut c_void, u32) -> u32 # EXPERIMENTAL fn omega_gpu_pf_qgemm fn(u32, u32, u32, u64, u32, u64, u64, u64, u64, *mut u64) -> i32 # EXPERIMENTAL +fn omega_gpu_pf_qgemm_scoped_bm fn(u32, u32, u32, u32) -> u32 # EXPERIMENTAL +fn omega_gpu_pf_qgemm_scoped fn(u32, u32, u32, u32, u32, u32, u32, u64, u32, u64, u64, u64, u64, *mut u64) -> i32 # EXPERIMENTAL fn omega_gpu_pf_qgemm_set_shape fn(u32, u32, u32) -> () # EXPERIMENTAL +fn omega_gpu_pf_qgemm_set_group fn(u32) -> () # EXPERIMENTAL fn omega_gpu_pf_quant_record fn(*mut OmegaGpuCmdList, u32, u32, u32, u64, u64, u64) -> i32 # EXPERIMENTAL fn omega_gpu_pf_qgemm_record fn(*mut OmegaGpuCmdList, u32, u32, u32, u64, u32, u64, u64, u64, u64) -> i32 # EXPERIMENTAL fn omega_gpu_pf_router_rows fn() -> u32 # EXPERIMENTAL @@ -1091,3 +1095,7 @@ const OMEGA_GEMM_ROUTE_WIDEN 2 # EXPERIMENTAL const OMEGA_GEMM_ROUTE_BF16X3 3 # EXPERIMENTAL const OMEGA_GEMM_ROUTE_FAST_BF16 4 # EXPERIMENTAL const OMEGA_GEMM_ROUTE_FFMA 5 # EXPERIMENTAL +fn omega_gpu_pf_qgemm_reading_option fn(u32, u32) -> u32 # EXPERIMENTAL +fn omega_gpu_pf_qgemm_workspace_fits fn(u64, u64, u64, u64, u64) -> i32 # EXPERIMENTAL + +fn omega_gpu_prefill_attn256_scoped fn(*mut OmegaGpuDecodeAttn256Cache, u32, u32, u32, *const f32, f32, f32, *mut f32, *mut OmegaGpuPrefillAttn256Info, u32, *mut u64) -> i32 # EXPERIMENTAL diff --git a/docs/runtime/portability_table.tsv b/docs/runtime/portability_table.tsv index 50eb4c3c..e9820ec6 100644 --- a/docs/runtime/portability_table.tsv +++ b/docs/runtime/portability_table.tsv @@ -62,6 +62,7 @@ src/omega_bw_em_q.h CHIP_SPECIFIC Private emitters for the FP8 QMMA placeholders src/omega_bw_hazard.c CHIP_SPECIFIC Static scoreboard hazard checker over encoded sm_121 kernel bytes (RAW, WAR, WAW against the Blackwell write and read barriers); decodes the operand positions the encoder writes. Includes stdlib.h. HOSTED_LIBC src/omega_bw_hazard.h CHIP_SPECIFIC Interface of the sm_121 scoreboard hazard checker; includes only stddef.h and stdint.h. NO_HOSTED_INCLUDE src/omega_bw_reconv.h CHIP_SPECIFIC Warp reconvergence regions (BSSY, BSYNC) for the Blackwell IR emitters. NO_HOSTED_INCLUDE +src/omega_gpu_attention_mirror.h CHIP_SPECIFIC Scoped native attention history placement; declares GPU-private cached-copy behavior and includes omega_gpu_prefill_attn256_api.h. NO_HOSTED_INCLUDE src/omega_gpu_attention_api.c CHIP_SPECIFIC Builds and launches its own sm_121 attention kernels (stall counts as nvcc writes them for sm_121) through the session: the tuned head_dim 64/128 kernels and, since runtime API 0.6.0, the general fallback kernel (head_dim 1..256, any GQA ratio). In the freestanding closure. FREESTANDING_PROVEN src/omega_gpu_attention_api.h CHIP_SPECIFIC Attention API; its kernel type OmegaBlackwellKernel comes from omega_blackwell_matmul.h. NO_HOSTED_INCLUDE src/omega_gpu_bf16_gemv_api.c CHIP_SPECIFIC Launches the bf16 GEMV through a command list; uses the Blackwell bf16 generator. NO_HOSTED_INCLUDE diff --git a/mk/attention-mirror.mk b/mk/attention-mirror.mk new file mode 100644 index 00000000..4fe1fbdf --- /dev/null +++ b/mk/attention-mirror.mk @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +CARRIED_HOST_TESTS += test-attention-mirror-policy +.PHONY: test-attention-mirror-policy +$(OUT_DIR)/attention-mirror-policy.so: src/cudapp_om_attn_mirror_plan.c src/cudapp_om_attn_mirror_step.c + @mkdir -p $(OUT_DIR) + $(CC) $(filter-out -MMD -MP,$(CFLAGS)) -shared -fPIC $^ -o $@ +test-attention-mirror-policy: $(OUT_DIR)/attention-mirror-policy.so + python3 om/attention-mirror/check.py $(abspath $(OUT_DIR))/attention-mirror-policy.so +ifeq ($(CUDAPP_OMC_HOST),1) + OMEGA_OMC="$(OMEGA_OMC)" OMEGA_BRIDGE="$(OMEGA_BRIDGE)" CC="$(CC)" python3 om/attention-mirror/mutants.py +endif + +CARRIED_HOST_TESTS += test-attention-mirror-dispatch +.PHONY: test-attention-mirror-dispatch +ATTN_MIRROR_WRAP = omega_gpu_session_lock omega_gpu_session_unlock omega_gpu_session_open omega_gpu_session_on_close omega_gpu_decode_attn256_cache_view omega_gpu_session_sync_to_gpu omega_gpu_session_sync_to_cpu omega_gpu_session_free omega_gpu_pf_alloc_dev_locked omega_gpu_pf_copy_va_locked omega_gpu_session_launch +$(OUT_DIR)/attention-mirror-dispatch: om/attention-mirror/dispatch-check.c src/omega_gpu_prefill_attn256_api.c $(OUT_DIR)/libomega_gpu.a + $(CC) $(CFLAGS) -o $@ $< $(OUT_DIR)/libomega_gpu.a $(foreach f,$(ATTN_MIRROR_WRAP),-Wl,--wrap=$(f)) -lpthread -lm +test-attention-mirror-dispatch: $(OUT_DIR)/attention-mirror-dispatch + $(abspath $(OUT_DIR))/attention-mirror-dispatch + CC="$(CC)" python3 om/attention-mirror/dispatch-mutants.py $(abspath $(OUT_DIR))/libomega_gpu.a diff --git a/mk/carried-tests.mk b/mk/carried-tests.mk index 7bdc6e2d..853f2e81 100644 --- a/mk/carried-tests.mk +++ b/mk/carried-tests.mk @@ -65,5 +65,6 @@ test-gpu-isolation: $(GPU_ISOLATION_TEST) # Chip-only binaries: built here so the host suite at least compiles them. test-carried-bins: $(GPU_DATTN256_MULTI_TEST) $(GPU_GDN_MULTI_TEST) $(GPU_PATTN256_TEST) $(GPU_EXACT_HEAD_TEST) -CARRIED_HOST_TESTS = test-bw-encode-control test-gpu-decode-attn256 test-gpu-gdn test-gpu-q38-graph-host \ +# Other fragments may register tests before this file in the wildcard include order. +CARRIED_HOST_TESTS += test-bw-encode-control test-gpu-decode-attn256 test-gpu-gdn test-gpu-q38-graph-host \ test-gpu-streams test-gpu-isolation test-carried-bins diff --git a/mk/prefill-shape.mk b/mk/prefill-shape.mk new file mode 100644 index 00000000..113f0ab5 --- /dev/null +++ b/mk/prefill-shape.mk @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +CARRIED_HOST_TESTS += test-prefill-shape +.PHONY: test-prefill-shape +$(OUT_DIR)/prefill-shape.so: src/cudapp_om_prefill_shape.c + @mkdir -p $(OUT_DIR) + $(CC) $(filter-out -MMD -MP,$(CFLAGS)) -shared -fPIC $< -o $@ +$(OUT_DIR)/prefill-cache-check: om/prefill-shape/cache-check.c $(OUT_DIR)/libomega_gpu.a + $(CC) $(CFLAGS) -o $@ $< $(OUT_DIR)/libomega_gpu.a -Wl,--wrap=omega_gpu_session_alloc_code -lpthread -lm +test-prefill-shape: $(OUT_DIR)/prefill-shape.so $(OUT_DIR)/gpu_qgemm_test $(OUT_DIR)/prefill-cache-check + $(abspath $(OUT_DIR))/prefill-cache-check + python3 om/prefill-shape/check.py $(abspath $(OUT_DIR))/prefill-shape.so + $(abspath $(OUT_DIR))/gpu_qgemm_test --host-only + $(abspath $(OUT_DIR))/gpu_qgemm_test --sim +ifeq ($(CUDAPP_OMC_HOST),1) + OMEGA_OMC="$(OMEGA_OMC)" OMEGA_BRIDGE="$(OMEGA_BRIDGE)" CC="$(CC)" python3 om/prefill-shape/mutants.py +endif + +CARRIED_HOST_TESTS += test-prefill-order +.PHONY: test-prefill-order +$(OUT_DIR)/prefill-order.so: src/cudapp_om_prefill_order.c + @mkdir -p $(OUT_DIR) + $(CC) $(filter-out -MMD -MP,$(CFLAGS)) -shared -fPIC $< -o $@ +test-prefill-order: $(OUT_DIR)/prefill-order.so $(OUT_DIR)/gpu_qgemm_test + python3 om/prefill-shape/order-check.py $(abspath $(OUT_DIR))/prefill-order.so +ifeq ($(CUDAPP_OMC_HOST),1) + OMEGA_OMC="$(OMEGA_OMC)" OMEGA_BRIDGE="$(OMEGA_BRIDGE)" CC="$(CC)" python3 om/prefill-shape/order-mutants.py +endif + +CARRIED_HOST_TESTS += test-prefill-scoped +.PHONY: test-prefill-scoped +$(OUT_DIR)/prefill-scoped-check: om/prefill-shape/scoped-check.c $(OUT_DIR)/libomega_gpu.a + $(CC) $(CFLAGS) -o $@ $< $(OUT_DIR)/libomega_gpu.a -Wl,--wrap=omega_gpu_pf_open_locked -Wl,--wrap=omega_gpu_pf_kernel_locked -lpthread -lm +test-prefill-scoped: $(OUT_DIR)/prefill-scoped-check + $(abspath $(OUT_DIR))/prefill-scoped-check + CC="$(CC)" python3 om/prefill-shape/scoped-mutants.py $(abspath $(OUT_DIR))/libomega_gpu.a + +CARRIED_HOST_TESTS += test-prefill-reading +.PHONY: test-prefill-reading +$(OUT_DIR)/prefill-reading.so: src/cudapp_om_prefill_reading.c + @mkdir -p $(OUT_DIR) + $(CC) $(filter-out -MMD -MP,$(CFLAGS)) -shared -fPIC $< -o $@ +test-prefill-reading: $(OUT_DIR)/prefill-reading.so + python3 om/prefill-shape/reading-check.py $(abspath $(OUT_DIR))/prefill-reading.so +ifeq ($(CUDAPP_OMC_HOST),1) + OMEGA_OMC="$(OMEGA_OMC)" OMEGA_BRIDGE="$(OMEGA_BRIDGE)" CC="$(CC)" python3 om/prefill-shape/reading-mutants.py +endif + +CARRIED_HOST_TESTS += test-prefill-workspace +.PHONY: test-prefill-workspace +$(OUT_DIR)/prefill-workspace.so: src/cudapp_om_prefill_workspace.c + @mkdir -p $(OUT_DIR) + $(CC) $(filter-out -MMD -MP,$(CFLAGS)) -shared -fPIC $< -o $@ +test-prefill-workspace: $(OUT_DIR)/prefill-workspace.so + python3 om/prefill-shape/workspace-check.py $(abspath $(OUT_DIR))/prefill-workspace.so +ifeq ($(CUDAPP_OMC_HOST),1) + OMEGA_OMC="$(OMEGA_OMC)" OMEGA_BRIDGE="$(OMEGA_BRIDGE)" CC="$(CC)" python3 om/prefill-shape/workspace-mutants.py +endif diff --git a/mk/q38-head.mk b/mk/q38-head.mk new file mode 100644 index 00000000..3ec5c28c --- /dev/null +++ b/mk/q38-head.mk @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Host-only verification. The --chip switch is never passed by make test. +.PHONY: test-q38-head-bounds +ifeq ($(CUDAPP_OMC_HOST),1) +CARRIED_HOST_TESTS += test-q38-head-bounds +test-q38-head-bounds: $(OUT_DIR)/libomega_gpu.a + OMEGA_OMC="$(OMEGA_OMC)" Q38_GPU_OUT="$(abspath $(OUT_DIR))/q38-head" \ + Q38_GPU_LIB="$(OUT_DIR)/libomega_gpu.a" CC="$(CC)" sh om/q38-head/run.sh +else +test-q38-head-bounds: + @echo "test-q38-head-bounds: NOT_RUN (requires a runnable pinned AArch64 omc)" + @exit 2 +endif diff --git a/mk/seal-reuse.mk b/mk/seal-reuse.mk new file mode 100644 index 00000000..3b6b8fd1 --- /dev/null +++ b/mk/seal-reuse.mk @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +CARRIED_HOST_TESTS += test-seal-reuse +# Integer-only generated Omega unit. Use distinct OUT_DIRs when comparing settings. +CUDAPP_SEAL_OPT ?= -O3 +$(OUT_DIR)/cudapp_om_closure_reseal.o: private CFLAGS += $(CUDAPP_SEAL_OPT) +.PHONY: test-seal-reuse +$(OUT_DIR)/seal-reuse.so: src/cudapp_om_closure_reseal.c + @mkdir -p $(OUT_DIR) + $(CC) $(filter-out -MMD -MP,$(CFLAGS)) $(CUDAPP_SEAL_OPT) -shared -fPIC $< -o $@ +test-seal-reuse: $(OUT_DIR)/seal-reuse.so + python3 om/pipeline/seal-reuse-check.py $(abspath $(OUT_DIR))/seal-reuse.so +ifeq ($(CUDAPP_OMC_HOST),1) + OMEGA_OMC="$(OMEGA_OMC)" OMEGA_BRIDGE="$(OMEGA_BRIDGE)" CC="$(CC)" CUDAPP_SEAL_OPT="$(CUDAPP_SEAL_OPT)" python3 om/pipeline/seal-reuse-mutants.py +endif diff --git a/nvrm/nvrm.c b/nvrm/nvrm.c index 663e5b70..127435b6 100644 --- a/nvrm/nvrm.c +++ b/nvrm/nvrm.c @@ -56,13 +56,15 @@ static int fail(Nvrm *rm, const char *fmt, ...); /* Pure class admission lives in seeded Omega. Class IDs are vendor constants: * open-gpu-doc clc7c0.h, clc56f.h, and vendored clc361.h / clc661.h. */ -static int select_classes(Nvrm *rm, const uint32_t *classes, uint32_t count) { +static int select_classes(Nvrm *rm, const uint32_t *classes, uint32_t count, + uint16_t vendor_id, uint16_t device_id) { rm->compute_class = rm->gpfifo_class = rm->usermode_class = 0; rm->target_sm = 0; rm->discrete = 0; uint64_t m[16] = {1, rm->sm_version, NV2080_CTRL_GR_INFO_SM_VERSION_8_06, NV2080_CTRL_GR_INFO_SM_VERSION_12_01, OMEGA_AMPERE_COMPUTE_B, OMEGA_AMPERE_GPFIFO_A, VOLTA_USERMODE_A, - BLACKWELL_COMPUTE_B, BLACKWELL_CHANNEL_GPFIFO_A, HOPPER_USERMODE_A}; + BLACKWELL_COMPUTE_B, BLACKWELL_CHANNEL_GPFIFO_A, HOPPER_USERMODE_A, + 0, 0, NV2080_CTRL_GR_INFO_SM_VERSION_10_04, vendor_id, device_id}; /* Marshal the vendor's class-list presence, without selecting a target. */ for (uint32_t i = 0; i < count; i++) { for (uint32_t k = 0; k < 3; k++) { @@ -427,7 +429,8 @@ int nvrm_open(Nvrm *rm) { if (cl.numClasses > 512) return fail(rm, "too many classes"); cl.classList = NV_PTR_TO_NvP64(classes); if (rm_control(rm, rm->device, NV0080_CTRL_CMD_GPU_GET_CLASSLIST, &cl, sizeof cl)) return -1; - if (cl.numClasses > 512 || select_classes(rm, classes, cl.numClasses)) return -1; + if (cl.numClasses > 512 || select_classes(rm, classes, cl.numClasses, + cards[found].pci_info.vendor_id, cards[found].pci_info.device_id)) return -1; if (rm_alloc(rm, rm->subdevice, rm->usermode_class, NULL, 0, &rm->usermode)) return -1; void *mmio = map_to_cpu(rm, rm->usermode, 0x10000, NULL, 0, 0); if (mmio == MAP_FAILED) return -1; diff --git a/om/attention-mirror/.crumb b/om/attention-mirror/.crumb new file mode 100644 index 00000000..c221c387 --- /dev/null +++ b/om/attention-mirror/.crumb @@ -0,0 +1,20 @@ +{ + "extensions": { + "generated": { + "children": [], + "children_root": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "compiled_at": "2026-11-01T22:59:52Z", + "compiler": "crumb-compile/1", + "digest": "sha256:d945c7e493aceb0c4db2d77ad277e777d0b7aca26704247439187df02febbc56", + "evidence_files": 0, + "evidence_root": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "files": 11, + "generated_at_commit": "245122b1fc2f36e65d6ece23399179eeea22b34f", + "source_tree": "sha256:2ad25d88f25e9abde53b73a4add0af1e11c14b67bf08e7a5071df44b4784021e" + } + }, + "layer": "native-policy", + "name": "attention-mirror", + "purpose": "Checked Omega admission and copy ordering for scoped GPU-cached attention history", + "schema_version": "1.0.0" +} diff --git a/om/attention-mirror/README.md b/om/attention-mirror/README.md new file mode 100644 index 00000000..691a8d6a --- /dev/null +++ b/om/attention-mirror/README.md @@ -0,0 +1,15 @@ +# Scoped attention history mirror + +This candidate carries the existing 30B GPU-cached history mechanism into 27B attention. Omega owns request admission, bounded allocation reuse and the operation sequence. Existing native transport allocates device-private memory and runs the existing copy and attention kernels. It adds no attention arithmetic, reduction-order change, dependency or persistent-prefix validity assumption. + +`plan.om` accepts an exact seven-element request: mode, query heads, K/V heads, capacity in positions, starting position, row count and bytes per K/V plane. Zero means bypass, all-ones means refusal, and another value is the required mirror size. Bounds and geometry are checked before multiplication. The layout constants come from `omega_gpu_decode_attn256_api.h` and `omega_blackwell_codegen_attn256.h`, including the tail slack; they are implementation contracts, not claimed chip limits. + +`step.om` directs allocation when capacity is insufficient, PREP, full K/V copy and ATTEND. Each operation must report successful completion before the next action is selected. A failure prevents attention or output publication. One buffer is reused across layers and requests, and released by the existing attention cleanup/session-close path. The CPU never reads or writes it. Original cache contents remain authoritative for decode and saved state. + +The new per-call entry has modes 0 (ordinary), 1 (mirror all supported requests) and 2 (provisional threshold at 1024 rows). Mode 2 avoids copying on tiny calls. This threshold needs further performance qualification. No process-global selector or deployed default changes. Existing host, device and recorded calls keep their behavior. + +The integrated production candidate passes 63 physical attention checks, six complete-score cases, seven threshold and tail cases, and a 32,768-token read followed by a short request. Every compared score bit matches. The complete native host suite now passes all 83 manifest targets, including both attention checks; Core's full preflight also passes. The registration repair preserves targets contributed by earlier build fragments. The first full-suite attempts caught a missing portability row and an omitted pinned Bridge path in the external runner. Both failures and their repairs are preserved. + +The production long read measured 109.500 seconds before and 103.451 seconds with cached history, including allocation and copy cost. The same immutable server passed short correctness, streaming, seeded sampling, and paired 32,768-token retrieval and continuation checks. First visible response was 108.372 seconds before and 104.728 seconds with the candidate; fresh follow-up was 108.918 versus 103.074 seconds. The repeated 5,017-token full-model total was essentially unchanged. These are single shared-machine observations, not quiet performance qualification. The 65,536-token candidate serving check also passes retrieval and saved/fresh continuation parity; first visible response was 237.355 seconds and saved follow-up 0.531 seconds. This is a candidate-only correctness observation, not a paired speed claim. The 128,000-token serving check also passes retrieval and saved/fresh continuation parity: first visible response was 570.312 seconds, saved follow-up 0.778 seconds, and fresh follow-up 570.967 seconds. [128K evidence](serving128-validation.json) pins the completed run. All planned short/32K/64K/128K serving correctness gates now pass. Quiet alternating timing and independent review remain open; no default promotion or deployment. [Follow-up evidence](followup-validation.json) records the complete 83-target suite and 64K check. [Integration evidence](integration-validation.json) pins the checked sources, binaries, receipts, original log hashes and remaining work; it supersedes the pending status in the historical targeted host receipt. + +Run `make test-attention-mirror-policy test-attention-mirror-dispatch` with the repository's pinned Omega compiler and Bridge. The independent host oracle checks admission and transitions, including malformed inputs and failed completion; source mutations challenge geometry, bounds, short-call bypass, both-plane copy, capacity and ordering. Python is external verification only. The actual native dispatch instrument substitutes fake device transport to inspect the executed sequence and its failure paths without opening a GPU. diff --git a/om/attention-mirror/check.py b/om/attention-mirror/check.py new file mode 100644 index 00000000..9d785cf5 --- /dev/null +++ b/om/attention-mirror/check.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Independent external oracle. Not a production dependency. +import ctypes,itertools,random,sys +lib=ctypes.CDLL(sys.argv[1]);U=ctypes.c_uint64;bad=2**64-1 +plan=lib.omega_native_attnmirplan;plan.restype=ctypes.c_uint32;plan.argtypes=[ctypes.POINTER(U),ctypes.POINTER(U),U] +step=lib.omega_native_attnmirnext;step.restype=ctypes.c_uint32;step.argtypes=[ctypes.POINTER(U)]+[U]*4 + +def expected(a): + mode,qh,kh,limit,pos,rows,plane=a + if not (mode in (0,1,2) and qh==24 and kh==4 and 0=1024) else 0 + +cases=[] +for limit in (1,30,1023,1024,4096,40960,131072): + for rows in (1,30,1023,1024,1025,4096,32768): + for pos in (0,max(0,limit-rows),limit): + for mode in (0,1,2,3):cases.append((mode,24,4,limit,pos,rows,limit*2048+8256)) +base=(2,24,4,40960,0,4096,40960*2048+8256) +for i in range(7): + for v in (0,1,3,23,25,1023,1024,32768,32769,131072,131073,2**32,2**64-1): + a=list(base);a[i]=v;cases.append(tuple(a)) +rng=random.Random(710) +for _ in range(1000): + limit=rng.randrange(1,131073);rows=rng.randrange(1,min(limit,32768)+1);pos=rng.randrange(limit-rows+1) + cases.append((rng.randrange(3),24,4,limit,pos,rows,limit*2048+8256)) +for a in cases: + out=U(99);assert plan(ctypes.byref(out),(U*7)(*a),7)==0;assert out.value==expected(a),('plan',a,out.value,expected(a)) +assert plan(None,(U*7)(*base),7)==32 +assert plan(ctypes.byref(U()),None,7)==32 +assert plan(ctypes.byref(U()),(U*7)(*base),6)==32 + +def next_expected(phase,need,have,ok): + if ok!=1 or (need and (need<20608 or need>536887424 or need%16)):return 0 + if phase==0:return 1 if need>have else 2 + if phase==1:return 2 if have>=need else 0 + if phase==2:return 3 if need else 4 + if phase==3:return 4 if need else 0 + if phase==4:return 5 + return 0 +n=0 +for a in itertools.product(range(7),(0,16,20608,20609,33570944,536887424,536887440,2**64-1),(0,16,20608,33570944,536887424,2**64-1),(0,1,2)): + out=U(99);assert step(ctypes.byref(out),*a)==0;assert out.value==next_expected(*a),('step',a,out.value,next_expected(*a));n+=1 +assert step(None,0,20608,0,1)==32 +print('PASS independent attention mirror oracle',len(cases),'plans',n,'transitions') diff --git a/om/attention-mirror/dispatch-check.c b/om/attention-mirror/dispatch-check.c new file mode 100644 index 00000000..3c975615 --- /dev/null +++ b/om/attention-mirror/dispatch-check.c @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +/* Host-only integration instrument: execute the actual adapter with fake device transport. */ +#include +#include +#include +#include +#include "../../src/omega_gpu_prefill_attn256_api.c" + +static int depth, n, fail, allocs, copies; +static unsigned events[16]; +static OmegaGpuDecodeAttn256CacheView cv; +static OmegaGpuSession fake_session; +static unsigned current_m; +static void event(unsigned x) { assert(depth == 1 && n < 16); events[n++] = x; } +void __wrap_omega_gpu_session_lock(void) { assert(depth++ == 0); } +void __wrap_omega_gpu_session_unlock(void) { assert(--depth == 0); } +OmegaGpuSession *__wrap_omega_gpu_session_open(void) { assert(depth == 1); return &fake_session; } +int __wrap_omega_gpu_session_on_close(void (*f)(void)) { (void)f; return fail == 9 ? -1 : 0; } +int __wrap_omega_gpu_decode_attn256_cache_view(const OmegaGpuDecodeAttn256Cache *c, uint32_t layer, OmegaGpuDecodeAttn256CacheView *v) { + assert(c && layer == 0); *v = cv; return 0; +} +int __wrap_omega_gpu_session_sync_to_gpu(const OmegaGpuMem *m) { assert(depth == 1 && m->cpu); return 0; } +int __wrap_omega_gpu_session_sync_to_cpu(const OmegaGpuMem *m) { assert(depth == 1 && m->cpu); return 0; } +void __wrap_omega_gpu_session_free(OmegaGpuMem *m) { assert(depth == 1); free(m->cpu); memset(m, 0, sizeof *m); } +int __wrap_omega_gpu_pf_alloc_dev_locked(size_t bytes, OmegaGpuMem *m) { + event(1); allocs++; + assert(bytes == 2*(cv.v_va-cv.k_va)); + if (fail == 1) return -1; + memset(m, 0, sizeof *m); m->va = 0x100123450ull; m->size = bytes; return 0; +} +int __wrap_omega_gpu_pf_copy_va_locked(uint64_t dst, uint64_t src, size_t bytes, uint64_t *ns) { + event(3); copies++; + assert(n >= 2 && events[n-2] == 2); + assert(dst == g.mirror.va && src == cv.k_va && bytes == 2*(cv.v_va-cv.k_va)); + if (ns) *ns = 17; + return fail == 3 ? -1 : 0; +} +static uint64_t ptr(const uint32_t *a, unsigned i) { return a[i] | (uint64_t)a[i+1]<<32; } +int __wrap_omega_gpu_session_launch(const OmegaGpuLaunch *l, uint64_t *ns, uint32_t *marker) { + (void)marker; + unsigned action = l->n_args == OMEGA_DATTN256_ARGS_WORDS ? 2 : 4; + event(action); + if (ns) *ns = action; + if (action == 2) { assert(ptr(l->args,2) == cv.k_va && ptr(l->args,4) == cv.v_va); } + else { + assert(l->n_args == OMEGA_PATTN256_ARGS_WORDS); + const int copied = n >= 2 && events[n-2] == 3; + assert(ptr(l->args,2) == (copied ? g.mirror.va : cv.k_va)); + assert(ptr(l->args,4) == (copied ? g.mirror.va + cv.v_va-cv.k_va : cv.v_va)); + for (size_t i = 0; i < (size_t)current_m*cv.out_words; i++) ((float *)g.out.cpu)[i] = 1.0f; + } + return fail == (int)action ? -1 : 0; +} +static void run(unsigned mode, unsigned m, const unsigned *want, size_t count, int failure) { + float *q = calloc((size_t)m*cv.qkv_words,4), *out = calloc((size_t)m*cv.out_words,4); + assert(q && out); n = 0; fail = failure; current_m = m; uint64_t copy_ns = 99; + OmegaGpuPrefillAttn256Info info; + int rc = omega_gpu_prefill_attn256_scoped((void *)1,0,0,m,q,1e-6f,1.0f,out,&info,mode,©_ns); + assert(depth == 0); assert((rc != 0) == (failure != 0)); assert((size_t)n == count); + for (size_t i = 0; i < count; i++) assert(events[i] == want[i]); + if (!rc) { assert(out[0] == 1.0f && out[(size_t)m*cv.out_words-1] == 1.0f); assert(copy_ns == ((mode==1 || (mode==2 && m>=1024)) ? 17u : 0u)); } + free(q); free(out); +} +int main(void) { + cv.max_pos=8192;cv.q_heads=24;cv.kv_heads=4;cv.q_per_cta=2;cv.qkv_words=8192;cv.out_words=6144; + cv.k_va=0x200345670ull;cv.v_va=cv.k_va+(uint64_t)cv.max_pos*4*OMEGA_DATTN256_ROW_BYTES+OMEGA_DATTN256_PLANE_SLACK; + uint32_t sh[3]={cv.q_heads,cv.kv_heads,cv.q_per_cta}; + for(unsigned k=0;k +#include +static unsigned allocations; +int __wrap_omega_gpu_session_alloc_code(size_t bytes, OmegaGpuMem *out) { + (void)bytes; (void)out; allocations++; return -1; +} +static int gen(const OmegaPfKey *key, void *vp) { + BlackwellIRProgram *p = vp; + OmegaQgSpec s; memset(&s, 0, sizeof s); + s.k=5120; s.n=16384; s.rt=key->words[0]; s.warps=8; s.wait=45; s.direct=1; + int rc=omega_blackwell_codegen_qgemm_prog(&s,p); + if (!rc && key->words[1]==1) p->regalloc.gpr_limit=0; + if (!rc && key->words[1]==2) p->regalloc.gpr_limit=BW_PHYS_GPR_HW_LIMIT+1; + if (!rc && key->words[1]==3) p->regalloc.peak_gpr_usage=BW_PHYS_GPR_HW_LIMIT+1; + return rc; +} +int main(void) { + unsigned failed=0; + for(unsigned i=0;i<5;i++) { + OmegaPfKey key; memset(&key,0,sizeof key); key.kind=999; + key.words[0]=i==0?4:8; key.words[1]=i>1?i-1:0; + OmegaPfKernel out; allocations=0; + int rc=omega_gpu_pf_kernel_locked(&key,gen,&out); + unsigned want=i<2; + if(allocations!=want || rc!=(want?OMEGA_GPU_PF_CHIP_FAIL:OMEGA_GPU_PF_TOO_LARGE)) { + fprintf(stderr,"cache case %u rc=%d allocations=%u wanted=%u\n",i,rc,allocations,want); failed++; + } + } + printf("prefill cache admission: %s\n",failed?"FAIL":"PASS"); + return failed?1:0; +} diff --git a/om/prefill-shape/cache-full-host-validation.json b/om/prefill-shape/cache-full-host-validation.json new file mode 100644 index 00000000..24d842d3 --- /dev/null +++ b/om/prefill-shape/cache-full-host-validation.json @@ -0,0 +1,16 @@ +{ + "status": "PASS", + "runtime_source_commit": "572a73bfd138b78154172f7746504842a7168f9c", + "targets": 77, + "exit_code": 0, + "started": "2026-11-01T19:45:51.631716+00:00", + "ended": "2026-11-01T19:54:34.887727+00:00", + "command": [ + "make", + "-j4", + "OUT_DIR=../optimize27-prefill128-fixed-build", + "test" + ], + "log_sha256": "748502861fb0d8189164c443cca81c0c0816fa8eb1cda5b569837608d1d74fd1", + "scope": "Complete local host suite at the named runtime source. Not server performance qualification." +} diff --git a/om/prefill-shape/check.py b/om/prefill-shape/check.py new file mode 100644 index 00000000..38b27b31 --- /dev/null +++ b/om/prefill-shape/check.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# External admission oracle, not a production dependency. +import ctypes,sys +lib=ctypes.CDLL(sys.argv[1]);f=lib.omega_native_qgshape +f.argtypes=[ctypes.POINTER(ctypes.c_uint64)]+[ctypes.c_uint64]*4;f.restype=ctypes.c_uint32 +cases=[(4,8,1),(2,4,1),(0,0,0),(1,2,0),(2,8,0),(4,4,0),(4,16,1),(8,16,0),(8,8,1),(8,4,0),(8,32,0),(2,0,0),(0,4,0),(2,2,0),(16,32,0),(2**32-1,4,0),(2,2**32-1,0),(2**64-1,4,0),(4,2**64-1,0)] +cases += [(rt,warps,int((rt,warps) in {(4,8),(4,16),(2,4),(8,8)})) for rt in range(17) for warps in range(33)] +for rt,warps,want in cases: + out=ctypes.c_uint64(999) + assert f(ctypes.byref(out),rt,warps,128,0)==0 + assert out.value==want,(rt,warps,out.value,want) +assert f(None,2,4,128,0)==32 +print('PASS shape admission and unsafe-shape refusal controls') + +# Independent exact-integer enumeration of every emitted store immediate. +count=0 +for rt in [0,2,4,8,9,2**64-1]: + warps=4 if rt==2 else 8 + for epi in [0,1,2,2**64-1]: + widths={0,1,128,5120,6144,16384,17408,34816,2**32-1,2**32,2**64-1} + if 1<=rt<=8 and epi in (0,1): + top=((1<<23)-1-(36 if epi==0 else 0))//((16*rt-8)*(4 if epi==0 else 1)) + widths.update([top-1,top,top+1]) + for n in sorted(widths): + valid=1<=n<2**32 and rt in (2,4,8) and epi in (0,1) + if valid: + stride=n*(4 if epi==0 else 1) + emitted=[(16*i+8*h)*stride+(32*j+4*q if epi==0 else 0) for i in range(rt) for h in range(2) for j in range(2) for q in range(2)] + want=int(all(0<=x<(1<<23) for x in emitted)) + else:want=0 + out=ctypes.c_uint64(999) + assert f(ctypes.byref(out),rt,warps,n,epi)==0 + assert out.value==want,(n,rt,epi,out.value,want) + count+=1 +assert f(None,8,8,16384,0)==32 +print('PASS exact emitted-offset boundary oracle',count,'cases') diff --git a/om/prefill-shape/large-shape-full-host-validation.json b/om/prefill-shape/large-shape-full-host-validation.json new file mode 100644 index 00000000..5aeb0a2f --- /dev/null +++ b/om/prefill-shape/large-shape-full-host-validation.json @@ -0,0 +1,15 @@ +{ + "status": "PASS_FULL_HOST_SUITE_GPU_NOT_RUN", + "tested_source_commit": "370c9b00bbd9f8b20efdbc3fa0cc844cb1d47b87", + "archive_sha256": "2372e067481f23ca7d8ef0126a900c1152ea1572d4e89ccb1c816ce433c6febe", + "command": "make -j4 OUT_DIR=../optimize27-prefill128-build test", + "execution": "CPUs12-15 at nice19; existing128K GPU qualification active; this suite opens no GPU", + "tool_configuration": "OMEGA_OMC and OMEGA_BRIDGE set to the versions pinned by om/units.sh and om/omega-bridge.lock", + "started_utc": "2026-11-01T19:25:38Z", + "ended_utc": "2026-11-01T19:33:46Z", + "exit_code": 0, + "final_line": "CUDA++ HOST TESTS PASS (77 targets)", + "log_sha256": "145212e485e6f8bcd5404e20c6c6bef5e85dc2c3106de59ef5ba7e8a23759f3f", + "earlier_attempt": "The first full-suite invocation used an absolute OUT_DIR. Existing recipes prefix ./ and could not execute that path. The corrected relative path names the same build directory. No test or production code was changed to obtain the pass.", + "limits": "Host validation only. No128-row chip/full-model parity or performance result yet. Default64rows unchanged." +} diff --git a/om/prefill-shape/large-shape-gpu-validation.json b/om/prefill-shape/large-shape-gpu-validation.json new file mode 100644 index 00000000..7ec173fd --- /dev/null +++ b/om/prefill-shape/large-shape-gpu-validation.json @@ -0,0 +1,87 @@ +{ + "status": "PASS_EXACT_SCORES", + "scope": "Shared-machine single-run diagnostic. Not quiet or statistically qualified.", + "formula": "seconds=ns/1e9; reduction_percent=100*(baseline_ns-candidate_ns)/baseline_ns", + "inputs": { + "reading-screen-group0-20261101T194419Z": "b8bcb76500664674cfc9ef99003f23df731a90fb121794abb79f6c0a581b2537", + "reading-screen-rows128fixed-20261101T194649Z": "3564f28863eb8f308decd70532fb7e094d85cb07d6cb1ebe2592ae776c616225", + "Optimize27-checkpoint-20261101": "44e4cb62fc06712f3728c26d0ba0a70dc4e516fb4557f80060f756720e8d26d6" + }, + "rows": [ + { + "trial": 0, + "prompt_tokens": 30, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "890aabe9fed6550d1665eaa1fb20c3dfc25efe514db8d8ca6b97160be5bde853", + "baseline_s": 0.332722031, + "candidate_s": 0.328368153, + "wall_reduction_percent": 1.308563183181579, + "baseline_gemm_s": 0.164627506, + "candidate_gemm_s": 0.170593682 + }, + { + "trial": 1, + "prompt_tokens": 513, + "chunk": 512, + "all_score_bits_and_next_equal": true, + "score_sha256": "751c59e6812c98bedc12ccb72bf878cb7c52fe6734a51a96646b1d0bd5937396", + "baseline_s": 2.258274006, + "candidate_s": 1.87290192, + "wall_reduction_percent": 17.064894914262233, + "baseline_gemm_s": 1.337981532, + "candidate_gemm_s": 0.924764204 + }, + { + "trial": 2, + "prompt_tokens": 941, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "36dabb2803481bdd0e552d3c631557e3047d2e233e43f5636dccceae143661c0", + "baseline_s": 3.650990357, + "candidate_s": 2.994418683, + "wall_reduction_percent": 17.98338559676453, + "baseline_gemm_s": 2.213912136, + "candidate_gemm_s": 1.491434136 + }, + { + "trial": 3, + "prompt_tokens": 5017, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "88ca539df5ecfc096a32e59aec5c48da86f9bcc54f6d78e2bcb9fa4a79535447", + "baseline_s": 19.126557271, + "candidate_s": 14.864002664, + "wall_reduction_percent": 22.28605256348436, + "baseline_gemm_s": 11.546749839, + "candidate_gemm_s": 7.405340297 + }, + { + "trial": 4, + "prompt_tokens": 30, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "890aabe9fed6550d1665eaa1fb20c3dfc25efe514db8d8ca6b97160be5bde853", + "baseline_s": 0.3078444, + "candidate_s": 0.304199012, + "wall_reduction_percent": 1.1841657668614405, + "baseline_gemm_s": 0.165410748, + "candidate_gemm_s": 0.173194736 + }, + { + "trial": 5, + "prompt_tokens": 5017, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "88ca539df5ecfc096a32e59aec5c48da86f9bcc54f6d78e2bcb9fa4a79535447", + "baseline_s": 18.816751006, + "candidate_s": 14.759224384, + "wall_reduction_percent": 21.563375211300812, + "baseline_gemm_s": 11.509962547, + "candidate_gemm_s": 7.419917663 + } + ], + "binary_sha256": "e2f9423accd4819a7657e2353b5af69b8a9a48be059a85ff299905711151dbbb", + "archive_sha256": "83faae846cad12f77bdbe8ff53bc84e58b267d3c89bab545308dacb781c1cf5b", + "source": "cache-admission-regression.json source identities" +} diff --git a/om/prefill-shape/large-shape-host-validation.json b/om/prefill-shape/large-shape-host-validation.json new file mode 100644 index 00000000..d4582561 --- /dev/null +++ b/om/prefill-shape/large-shape-host-validation.json @@ -0,0 +1,104 @@ +{ + "status": "HOST_SIMULATOR_PASS_GPU_NOT_RUN", + "native_parent": "2d7a41c0173c544630cc43d12eb9740bff4b8c4e", + "source_sha256": { + "om/prefill-shape/admit.om": "5720ae7a311677f31e065e8d68b8470dfb94e3d13424f41ef211a5ace163e46f", + "om/prefill-shape/check.py": "fd7ceab17cc2abcf66b89b9b261393735d0dd8a60ba72af06e2a278e095af966", + "om/prefill-shape/mutants.py": "8c40880bb2fb0f25898aa3fce75f8651abc614659e9ab3d451a6e1d9c7fb14a1", + "src/cudapp_native.h": "3ba96c9c0c9a6b18e4d1f08ac34fffa483c296a94980808f40162c911c99770e", + "src/cudapp_om_prefill_shape.c": "4dca73ff1821f73bdc6d13050a1a1be4762354ee9c6d082145adf057c49530b6", + "src/cudapp_om_prefill_shape.ident": "795a51ff3375bf0fc5ea52974957e85319ff837dbb8d48788e1b7af3f3f35d6c", + "src/cudapp_om_units.ident": "f1a152db38968a8819cef9d318343b7da895cdc8f36a757031fbdedc92ff0053", + "src/omega_blackwell_codegen_qg.c": "f30ecf540459b32eb6be229e0239d8df9b6d3ee4305814a48219cd0faa930ec2", + "src/omega_blackwell_codegen_qg.h": "fd13c4f148b651bc0dd0de20045e9642c74ba37930e5d9c7da5ef9f0e1109a67", + "tests/gpu_qgemm_test.c": "a769cfdde2f9321a5697497fe7c52d68a1e0735561432e91f03fd7c9b9c9ac2a" + }, + "archive_sha256": "2372e067481f23ca7d8ef0126a900c1152ea1572d4e89ccb1c816ce433c6febe", + "command": "make -j4 OUT_DIR=build test-prefill-shape", + "tool_configuration": "OMEGA_OMC is the compiler pinned by om/units.sh; OMEGA_BRIDGE is the source pinned by om/omega-bridge.lock. The recorded run used a separate output directory.", + "host_and_simulator_exit_code": 0, + "selected_output": [ + "PASS shape admission and unsafe-shape refusal controls", + "PASS exact emitted-offset boundary oracle 282 cases", + "PASS qgemm SwiGLU: MUFU flush vs IEEE, 15859204 inputs, 0 differ (t-cap keeps it unobservable, #462)", + "PASS 128-row shape 0 peak 144 declared 152 max-store 7864356", + "PASS 128-row shape 1 peak 144 declared 152 max-store 6881316", + "PASS 128-row shape 2 peak 144 declared 152 max-store 2457636", + "PASS 128-row shape 3 peak 145 declared 152 max-store 4177920", + "PASS 128-row shape 4 peak 144 declared 152 max-store 2457636", + "GPU_QGEMM: PASS", + "sim quantiser k 2048 f32 groups 0: ran, 0 differ", + "sim quantiser k 768 bf16 groups 0: ran, 0 differ", + "sim quantiser k 4096 f32 groups 0: ran, 0 differ", + "sim quantiser k 128 bf16 groups 0: ran, 0 differ", + "sim quantiser k 5120 f32 groups 20: ran, 0 differ", + "sim quantiser k 6144 bf16 groups 24: ran, 0 differ", + "sim quantiser k 17408 bf16 groups 17: ran, 0 differ", + "sim quantiser k 2048 f32 groups 4: ran, 0 differ", + "sim quantiser mutant 1: caught", + "sim quantiser mutant 2: caught", + "sim quantiser mutant 3: caught", + "sim qgemm case 0 (n 256 k 384 M 300 swiglu indexed rt 4 warps 8) order 0: ran, 0 differ", + "sim qgemm case 0 (n 256 k 384 M 300 swiglu indexed rt 4 warps 8) order 1: ran, 0 differ", + "sim qgemm case 0 mutant 1: caught", + "sim qgemm case 0 mutant 2: caught", + "sim qgemm case 0 mutant 3: caught", + "sim qgemm case 0 mutant 4: caught", + "sim qgemm case 1 (n 256 k 256 M 200 f32 direct rt 4 warps 8) order 0: ran, 0 differ", + "sim qgemm case 1 (n 256 k 256 M 200 f32 direct rt 4 warps 8) order 1: ran, 0 differ", + "sim qgemm case 1 mutant 1: caught", + "sim qgemm case 1 mutant 2: caught", + "sim qgemm case 1 mutant 3: caught", + "sim qgemm case 1 mutant 4: caught", + "sim qgemm case 2 (n 256 k 128 M 150 f32 indexed rt 4 warps 8) order 0: ran, 0 differ", + "sim qgemm case 2 (n 256 k 128 M 150 f32 indexed rt 4 warps 8) order 1: ran, 0 differ", + "sim qgemm case 3 (n 512 k 1024 M 140 swiglu direct rt 4 warps 8) order 0: ran, 0 differ", + "sim qgemm case 3 (n 512 k 1024 M 140 swiglu direct rt 4 warps 8) order 1: ran, 0 differ", + "sim qgemm case 4 (n 256 k 384 M 259 swiglu indexed rt 8 warps 8) order 0: ran, 0 differ", + "sim qgemm case 4 (n 256 k 384 M 259 swiglu indexed rt 8 warps 8) order 1: ran, 0 differ", + "sim qgemm case 4 mutant 1: caught", + "sim qgemm case 4 mutant 2: caught", + "sim qgemm case 4 mutant 3: caught", + "sim qgemm case 4 mutant 4: caught", + "sim qgemm case 5 (n 256 k 256 M 257 f32 direct rt 8 warps 8) order 0: ran, 0 differ", + "sim qgemm case 5 (n 256 k 256 M 257 f32 direct rt 8 warps 8) order 1: ran, 0 differ", + "sim qgemm case 5 mutant 1: caught", + "sim qgemm case 5 mutant 2: caught", + "sim qgemm case 5 mutant 3: caught", + "sim qgemm case 5 mutant 4: caught", + "sim qgemm case 6 (n 256 k 128 M 127 f32 indexed rt 8 warps 8) order 0: ran, 0 differ", + "sim qgemm case 6 (n 256 k 128 M 127 f32 indexed rt 8 warps 8) order 1: ran, 0 differ", + "sim qgemm case 7 (n 512 k 1024 M 129 swiglu direct rt 8 warps 8) order 0: ran, 0 differ", + "sim qgemm case 7 (n 512 k 1024 M 129 swiglu direct rt 8 warps 8) order 1: ran, 0 differ", + "sim qgemm case 8 (n 256 k 384 M 97 swiglu indexed rt 2 warps 4) order 0: ran, 0 differ", + "sim qgemm case 8 (n 256 k 384 M 97 swiglu indexed rt 2 warps 4) order 1: ran, 0 differ", + "sim qgemm case 8 mutant 1: caught", + "sim qgemm case 8 mutant 2: caught", + "sim qgemm case 8 mutant 3: caught", + "sim qgemm case 8 mutant 4: caught", + "sim qgemm case 9 (n 256 k 256 M 65 f32 direct rt 2 warps 4) order 0: ran, 0 differ", + "sim qgemm case 9 (n 256 k 256 M 65 f32 direct rt 2 warps 4) order 1: ran, 0 differ", + "sim qgemm case 9 mutant 1: caught", + "sim qgemm case 9 mutant 2: caught", + "sim qgemm case 9 mutant 3: caught", + "sim qgemm case 9 mutant 4: caught", + "sim qgemm case 10 (n 256 k 128 M 31 f32 indexed rt 2 warps 4) order 0: ran, 0 differ", + "sim qgemm case 10 (n 256 k 128 M 31 f32 indexed rt 2 warps 4) order 1: ran, 0 differ", + "sim qgemm case 11 (n 512 k 1024 M 30 swiglu direct rt 2 warps 4) order 0: ran, 0 differ", + "sim qgemm case 11 (n 512 k 1024 M 30 swiglu direct rt 2 warps 4) order 1: ran, 0 differ", + "GPU_QGEMM: PASS", + "drop_column_offset: caught", + "overflow_offset_boundary: caught", + "drop_last_row_group: caught", + "drop_large_shape: caught", + "admit_large_overbudget_warps: caught", + "drop_small_shape: caught", + "drop_incumbent: caught", + "admit_unsafe_one_row_group: caught", + "ignore_warp_count: caught" + ], + "full_host_suite": "RUNNING at publication; not a PASS claim", + "chip_and_model": "NOT_RUN: full-model probe built; waiting for the current128K qualification to finish", + "performance": "UNMEASURED: no default change or speed claim", + "original_default": "64rows,8warps,wait45 retained" +} diff --git a/om/prefill-shape/mutants.py b/om/prefill-shape/mutants.py new file mode 100644 index 00000000..3f810e82 --- /dev/null +++ b/om/prefill-shape/mutants.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# External source-mutant verification, not a production dependency. +import hashlib,os,subprocess,tempfile +from pathlib import Path +root=Path(__file__).resolve().parents[2] +source=(root/'om/prefill-shape/admit.om').read_text() +mutants={ + 'drop_column_offset':('last = add last 36','last = add last 0'), + 'overflow_offset_boundary':('le last 8388607','le last 8388643'), + 'drop_last_row_group':('sub mul rt 16 8','sub mul rt 16 16'), + 'drop_large_shape':('eq rt 8 eq warps 8','eq rt 7 eq warps 8'), + 'admit_large_overbudget_warps':('eq rt 8 eq warps 8','eq rt 8 eq warps 16'), + 'drop_small_shape':('eq rt 2 eq warps 4','eq rt 3 eq warps 4'), + 'drop_incumbent':('eq rt 4 eq warps 8','eq rt 5 eq warps 8'), + 'admit_unsafe_one_row_group':('eq rt 2 eq warps 4','eq rt 1 eq warps 2'), + 'ignore_warp_count':('eq rt 2 eq warps 4','eq rt 2 eq rt 2'), +} +ident=(root/'src/cudapp_om_prefill_shape.ident').read_text().strip() +pins=dict(part.split('=',1) for part in ident.split() if '=' in part) +assert hashlib.sha256(Path(os.environ['OMEGA_OMC']).read_bytes()).hexdigest()==pins['omc'] +env=dict(os.environ,OMEGA_SEEDED_OMC=os.environ['OMEGA_OMC']) +bridge=Path(os.environ['OMEGA_BRIDGE'])/'tools/bridge-seeded' +assert hashlib.sha256(bridge.read_bytes()).hexdigest()==pins['frontend'] +assert hashlib.sha256((bridge.parent.parent/'src/toc.om').read_bytes()).hexdigest()==pins['converter'] +with tempfile.TemporaryDirectory(prefix='prefill-shape-mutants-') as t: + t=Path(t) + for name,(old,new) in mutants.items(): + assert source.count(old)==1 + om=t/(name+'.om');om.write_text(source.replace(old,new)) + c=t/(name+'.c');so=t/(name+'.so') + with c.open('w') as f:subprocess.run(['sh',str(bridge),'c',str(om)],env=env,stdout=f,check=True) + subprocess.run([os.environ.get('CC','cc'),'-O2','-shared','-fPIC',str(c),'-o',str(so)],check=True) + p=subprocess.run(['python3',str(root/'om/prefill-shape/check.py'),str(so)],capture_output=True,text=True) + assert p.returncode!=0,name+' escaped oracle' + assert 'AssertionError' in p.stderr,(name,p.stderr) + print(name+': caught',flush=True) diff --git a/om/prefill-shape/order-check.py b/om/prefill-shape/order-check.py new file mode 100644 index 00000000..f8e5ec35 --- /dev/null +++ b/om/prefill-shape/order-check.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Independent external oracle. Not a production dependency. +import ctypes,itertools,sys +lib=ctypes.CDLL(sys.argv[1]);fn=lib.omega_native_qgorder +fn.argtypes=[ctypes.POINTER(ctypes.c_uint64),ctypes.c_void_p]+[ctypes.c_uint64]*5 +fn.restype=ctypes.c_uint32 + +def plan(group,cols,tiles,guard=1): + out=(ctypes.c_uint64*64)();result=ctypes.c_uint64(99) + rc=fn(ctypes.byref(result),out,64,group,cols,tiles,guard) + assert rc==0,rc + return result.value,list(out) + +def execute(p,x,y): + r=[x,y,0,0];pred=False + for i in range(p[2]): + op,d,a,b,imm=p[3+5*i:8+5*i] + assert max(d,a,b)<4 + if op==1:r[d]=r[a]&imm + elif op==2:r[d]=r[a]>>imm + elif op==3:r[d]=r[a]*imm+r[b] + elif op==4: + assert imm==896+11*4 + r[d]=execute.tiles + elif op==5:pred=r[a]>=r[b] + elif op==6: + if pred:return None + else:raise AssertionError('unknown opcode') + return (r[0],r[1]) + +def cover(p,cols,tiles): + execute.tiles=tiles;seen=set() + for x,y in itertools.product(range(p[0]),range(p[1])): + mapped=execute(p,x,y) + if mapped is None:continue + if mapped in seen or not (0<=mapped[0]=1 + om=t/(name+'.om');om.write_text(source.replace(old,new,1)) + c=t/(name+'.c');so=t/(name+'.so') + with c.open('w') as f:subprocess.run(['sh',str(bridge),'c',str(om)],env=env,stdout=f,check=True) + subprocess.run([os.environ.get('CC','cc'),'-O2','-shared','-fPIC',str(c),'-o',str(so)],check=True) + p=subprocess.run(['python3',str(root/'om/prefill-shape/order-check.py'),str(so)],capture_output=True,text=True) + assert p.returncode!=0,name+' escaped oracle' + assert 'AssertionError' in p.stderr,(name,p.stderr) + print(name+': caught',flush=True) diff --git a/om/prefill-shape/order-validation.json b/om/prefill-shape/order-validation.json new file mode 100644 index 00000000..f7cc9f5b --- /dev/null +++ b/om/prefill-shape/order-validation.json @@ -0,0 +1,132 @@ +{ + "status": "PASS_EXACT_SCORES", + "scope": "Shared-machine single-run diagnostic. Not quiet or statistically qualified.", + "formula": "seconds=ns/1e9; reduction_percent=100*(baseline_ns-candidate_ns)/baseline_ns", + "inputs": { + "reading-screen-group0-20261101T194419Z": "b8bcb76500664674cfc9ef99003f23df731a90fb121794abb79f6c0a581b2537", + "reading-screen-group4-20261101T194534Z": "a7802b188ee8d2bb54e0a8d4d9e3dcffaf50867f1fdb6b81dfd8e86be008ffc6", + "Optimize27-checkpoint-20261101": "44e4cb62fc06712f3728c26d0ba0a70dc4e516fb4557f80060f756720e8d26d6" + }, + "rows": [ + { + "trial": 0, + "prompt_tokens": 30, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "890aabe9fed6550d1665eaa1fb20c3dfc25efe514db8d8ca6b97160be5bde853", + "baseline_s": 0.332722031, + "candidate_s": 0.328841392, + "wall_reduction_percent": 1.1663306419285473, + "baseline_gemm_s": 0.164627506, + "candidate_gemm_s": 0.163969208 + }, + { + "trial": 1, + "prompt_tokens": 513, + "chunk": 512, + "all_score_bits_and_next_equal": true, + "score_sha256": "751c59e6812c98bedc12ccb72bf878cb7c52fe6734a51a96646b1d0bd5937396", + "baseline_s": 2.258274006, + "candidate_s": 2.156755011, + "wall_reduction_percent": 4.4954241482776025, + "baseline_gemm_s": 1.337981532, + "candidate_gemm_s": 1.209889835 + }, + { + "trial": 2, + "prompt_tokens": 941, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "36dabb2803481bdd0e552d3c631557e3047d2e233e43f5636dccceae143661c0", + "baseline_s": 3.650990357, + "candidate_s": 3.306518864, + "wall_reduction_percent": 9.435015141564232, + "baseline_gemm_s": 2.213912136, + "candidate_gemm_s": 1.795833972 + }, + { + "trial": 3, + "prompt_tokens": 5017, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "88ca539df5ecfc096a32e59aec5c48da86f9bcc54f6d78e2bcb9fa4a79535447", + "baseline_s": 19.126557271, + "candidate_s": 16.36954494, + "wall_reduction_percent": 14.414577029919688, + "baseline_gemm_s": 11.546749839, + "candidate_gemm_s": 8.809191353 + }, + { + "trial": 4, + "prompt_tokens": 30, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "890aabe9fed6550d1665eaa1fb20c3dfc25efe514db8d8ca6b97160be5bde853", + "baseline_s": 0.3078444, + "candidate_s": 0.316550316, + "wall_reduction_percent": -2.8280248073377328, + "baseline_gemm_s": 0.165410748, + "candidate_gemm_s": 0.162994709 + }, + { + "trial": 5, + "prompt_tokens": 5017, + "chunk": 4096, + "all_score_bits_and_next_equal": true, + "score_sha256": "88ca539df5ecfc096a32e59aec5c48da86f9bcc54f6d78e2bcb9fa4a79535447", + "baseline_s": 18.816751006, + "candidate_s": 16.198925508, + "wall_reduction_percent": 13.91220778318886, + "baseline_gemm_s": 11.509962547, + "candidate_gemm_s": 8.801004973 + } + ], + "binary_sha256": "8b8687ada95f10254ddf7dbedb9bbecc080fd35c42bed77d104666d417388d58", + "settings": { + "baseline": { + "rt": 4, + "group": 0 + }, + "candidate": { + "rt": 4, + "group": 4 + }, + "warps": 8, + "wait": 45 + }, + "host": { + "status": "PASS", + "exit_code": 0, + "targets": 78, + "command": "make -j4 OUT_DIR=../optimize27-qg-order-build test test-runtime-api-semver RUNTIME_API_BASE=ecb672c77f6f9145f0983348277ae8f58c74c8d3", + "compiler_sha256": "2c82480596bbdc50cda9626866209ef662d5834f278f839ff4ce0250c495029f", + "full_log_sha256": "4db1c5946321e1d77594196896039749f70246247dd202646de6bb8051ca52eb", + "source_identity": { + "base": "ecb672c77f6f9145f0983348277ae8f58c74c8d3", + "status": "PASS full host suite78, semver, exact GPU scores group0/group4; shared timing only", + "sources": { + "docs/runtime/API.md": "2d9e4e4390ca852ff05dd76e6290778cf90173e94348b8deabb3978ec5e1292f", + "docs/runtime/omega_runtime_api.manifest": "0fd414db619b7fc8d40b503c4368e415fd215b05e29039346a855e3d8e5d717e", + "mk/prefill-shape.mk": "ccba4c78dc228c864094028674319b6c07a6b403a852c7b0e10014805c30fe39", + "om/prefill-shape/README.md": "5cfd7bc8ac55bb62a19996a0c79de2eebecc0357121a362c2705dcf8157cc293", + "om/units.sh": "90e2b06d539b0fce4b8b1642985bd339f7df103c10689721bdf279cbb6ef6324", + "src/cudapp_native.h": "91cab62eed37af70fe6b83710b5c0d2ea1f365f4a626fb988550ea3a0c0a65d6", + "src/cudapp_om_units.ident": "77b16d01c829b52e115eb9ed659664921fa3d9d419319869e03b0399c8f43650", + "src/omega_blackwell_codegen_qg.c": "cf2c5cd3d58d9f0439e116143199054e0d6ff4c7117c96c64ad6b0995072cfeb", + "src/omega_blackwell_codegen_qg.h": "c19bbd7f882c99da69b23d2e6cb9b3fd999f541803730f0aa0db023019ed4806", + "src/omega_gpu_pfgemm.c": "c2379b1758dd402dc2eb5608404e5f333531994e39ebd5d7f8058247e21e47b7", + "src/omega_gpu_prefill.h": "3460cfeeb2f19799b318e059a6cdfb66447ab0a4dc720d8c5122e98d86c02e90", + "src/omega_runtime_api.h": "a52d32533dafe83ca20d69399b59f998389cd867cb3b9de6b4e2937926e3547f", + "tests/bw_hazard_test.c": "9cb312122fab1e0f9cfeb17a42dd64310d51919bf9fa6f019bc82f4e2ead9757", + "tests/gpu_qgemm_test.c": "407875e2d792c0db39903bf3a2689a954e90a5c2698426fb09d93ffe0108acd9", + "om/prefill-shape/order.om": "13f0e33cd7013f62564a82e5e6a2a9722ab3de92e10934ed46fffc2c5174de9b", + "om/prefill-shape/order-check.py": "62032077e16c8e0e99c32c016b051e19b706b887da2e722b04c63c7124ff73b4", + "om/prefill-shape/order-mutants.py": "0795f05f7661e0c0b58dd066cae8e470f64392b6389ea3b7a8d25e0aaf199442", + "src/cudapp_om_prefill_order.c": "43c835999dd230d3b953ad5b002a63d116638d93b4cb6893525676db6432ae77", + "src/cudapp_om_prefill_order.ident": "fb4a96a87f733878627afadf6562b9f6b3c51bea805cb9c61c51b33ab9bcce71" + }, + "archive_sha256": "7d29f388fd6e270e4017df99e7e2ef18f914a9aeee0837a74fc9b05980eb0cbd" + }, + "scope": "Host tests and simulated matrix arithmetic, including all 137 plan cases and five Omega source mutants. Not hardware performance qualification." + } +} diff --git a/om/prefill-shape/order.om b/om/prefill-shape/order.om new file mode 100644 index 00000000..f35e69b8 --- /dev/null +++ b/om/prefill-shape/order.om @@ -0,0 +1,53 @@ +note SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +note Grouped work order: adjacent column jobs reuse one weight panel across row tiles. +note Private plan: grid x, grid y, record count, then records of opcode,dst,src1,src2,immediate. +note Registers 0=cx,1=cy,2=temporary,3=tile count. Opcodes are private adapter operations. +fn qgo_put plan edit ulong 64 at ulong op ulong regs ulong imm ulong to ulong +put plan at op +put plan add at 1 rem regs 4 +put plan add at 2 rem div regs 4 4 +put plan add at 3 div regs 16 +put plan add at 4 imm +ret add at 5 +end + +fn qgo_emit plan edit ulong 64 group ulong guard ulong to ulong +size = shl ulong 1 group +at = qgo_put plan 3 1 2 sub size 1 +at = qgo_put plan at 2 0 group +at = qgo_put plan at 3 37 size +if eq guard 1 +note Tile count is constant-bank argument word11, byte896+44=940. +at = qgo_put plan at 4 3 940 +at = qgo_put plan at 5 52 0 +at = qgo_put plan at 6 0 0 +end +put plan 2 div sub at 3 5 +ret 1 +end + +fn native_qgorder plan edit ulong 64 group ulong cols ulong tiles ulong guard ulong to ulong +put plan 0 0 +put plan 1 0 +put plan 2 0 +if or gt group 5 gt guard 1 +ret 0 +end +if or eq cols 0 gt cols 65535 +ret 0 +end +if or eq tiles 0 gt tiles 65535 +ret 0 +end +size = shl ulong 1 group +gx = mul cols size +if gt gx 65535 +ret 0 +end +put plan 0 gx +put plan 1 div add tiles sub size 1 size +if eq group 0 +ret 1 +end +ret qgo_emit plan group guard +end diff --git a/om/prefill-shape/reading-check.py b/om/prefill-shape/reading-check.py new file mode 100644 index 00000000..7ceda8fb --- /dev/null +++ b/om/prefill-shape/reading-check.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# External profile oracle, not a production dependency. +import ctypes,sys +lib=ctypes.CDLL(sys.argv[1]) + +# Independent profile table at every small boundary and representative large rows. +r=lib.omega_native_qgreading +r.argtypes=[ctypes.POINTER(ctypes.c_uint64),ctypes.c_uint64,ctypes.c_uint64];r.restype=ctypes.c_uint32 +for rows in list(range(0,258))+[4095,4096,4097,2**32-1,2**32,2**64-1]: + for index in [0,1,2,3,4,2**64-1]: + out=ctypes.c_uint64(123) + assert r(ctypes.byref(out),rows,index)==0 + expected=(4,8,45,0) if 0 +#include +static unsigned opened, keyed, failed; +static OmegaQgSpec want; +int __wrap_omega_gpu_pf_open_locked(void) { opened++; return OMEGA_GPU_PF_OK; } +int __wrap_omega_gpu_pf_kernel_locked(const OmegaPfKey *key, OmegaPfGen gen, OmegaPfKernel *out) { + OmegaQgSpec got; (void)gen; (void)out; + memcpy(&got,key->words,sizeof got); keyed++; + if (memcmp(&got,&want,sizeof got)) { fprintf(stderr,"wrong per-call kernel key\n"); failed++; } + return OMEGA_GPU_PF_TOO_LARGE; /* stop before allocation/launch */ +} +static void expect(unsigned rt,unsigned warps,unsigned group,unsigned mutant) { + memset(&want,0,sizeof want);want.rt=rt;want.warps=warps;want.group=group;want.mutant=mutant; + want.k=128;want.n=256;want.wait=45;want.direct=1; + opened=keyed=0; +} +static void check(int ok,const char *why) { if(!ok) {failed++;fprintf(stderr,"%s\n",why);} } +static int scoped(unsigned rt,unsigned group) { + return omega_gpu_pf_qgemm_scoped(rt,8,45,group,256,128,OMEGA_GPU_PF_PLAIN,16,1,0,32,48,64,NULL); +} +int main(void) { + omega_gpu_pf_qgemm_set_shape(2,4,45);omega_gpu_pf_qgemm_set_group(1);omega_gpu_pf_qgemm_test_set_mutant(1); + check(omega_gpu_pf_qgemm_scoped_bm(8,8,45,4)==128,"checked128 tile height"); + check(omega_gpu_pf_qgemm_scoped_bm(4,8,45,0)==64,"checked64 tile height"); + check(omega_gpu_pf_qgemm_scoped_bm(0,8,45,4)==0,"zero shape refused"); + check(omega_gpu_pf_qgemm_scoped_bm(8,8,121,4)==0,"unsafe wait refused"); + check(omega_gpu_pf_qgemm_scoped_bm(8,8,45,6)==0,"group outside bound refused"); + for(unsigned i=0;i<3;i++) { + unsigned rt=i==1?4:8, group=i==1?0:4; + expect(rt,8,group,0); + check(scoped(rt,group)==OMEGA_GPU_PF_TOO_LARGE && opened==1 && keyed==1,"scoped call reached its own kernel key"); + expect(2,4,1,1); + int rc=omega_gpu_pf_qgemm(256,128,OMEGA_GPU_PF_PLAIN,16,1,0,32,48,64,NULL); + check(rc==OMEGA_GPU_PF_TOO_LARGE && opened==1 && keyed==1,"legacy settings unchanged after scoped call"); + check(omega_gpu_pf_qgemm_bm()==32,"legacy tile height unchanged"); + } + const uint32_t rows[] = {1,30,64,65,128,4096,UINT32_MAX}; + for(unsigned i=0;i 209 GB/s of distinct weights", + "q38gu mutant 1: caught", + "q38gu mutant 2: caught", + "q38gu mutant 3: caught", + "q38gu mutant 4: caught", + "q38down second launch identical: yes", + "q38down bit-identity on 3 sampled rows: 0 differ", + "QGEMM-inplace q38down tokens 30 M 30 n 5120 k 17408 tiles 1: median 0.454 ms (min 0.453), 11.8 TFLOPS, weights 89 MB -> 196 GB/s of distinct weights", + "q38down mutant 1: caught", + "q38down mutant 2: caught", + "q38down mutant 3: caught", + "q38down mutant 4: caught", + "q38mix second launch identical: yes", + "q38mix bit-identity on 3 sampled rows: 0 differ", + "QGEMM-inplace q38mix tokens 30 M 30 n 16384 k 5120 tiles 1: median 0.393 ms (min 0.389), 12.8 TFLOPS, weights 84 MB -> 213 GB/s of distinct weights", + "q38o second launch identical: yes", + "q38o bit-identity on 3 sampled rows: 0 differ", + "QGEMM-inplace q38o tokens 30 M 30 n 5120 k 6144 tiles 1: median 0.142 ms (min 0.140), 13.3 TFLOPS, weights 31 MB -> 221 GB/s of distinct weights" + ], + "limits": [ + "F32 chip reference is bit-exact on sampled rows. SwiGLU uses the incumbent one-BF16-ulp oracle tolerance.", + "The full output buffer is bit-identical across repeated launches for each case.", + "Simulator cases compare every output bit and include tails and direct/indexed layouts.", + "Full-model logits and token parity have NOT_RUN. Default remains unchanged.", + "Timings are shared-machine diagnostic data, not whole-model or quiet-machine qualification." + ] +} diff --git a/om/prefill-shape/workspace-check.py b/om/prefill-shape/workspace-check.py new file mode 100644 index 00000000..f45cb168 --- /dev/null +++ b/om/prefill-shape/workspace-check.py @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Independent capacity oracle, not production. +import ctypes,sys,random +lib=ctypes.CDLL(sys.argv[1]);f=lib.omega_native_qgworkspace +f.argtypes=[ctypes.POINTER(ctypes.c_uint64)]+[ctypes.c_uint64]*5;f.restype=ctypes.c_uint32 +cases=[] +for rows in [0,1,3,30,63,64,65,128,129,512,4096,2**32-1,2**32,2**64-1]: + for bm in [0,32,64,128,2**32-1,2**32,2**64-1]: + for delta in [-1,0,1]: + cap=max(0,rows+delta);pad=min(2**64-1,rows+bm+delta);tiles=(rows+bm-1)//bm+1 if bm else 0 + if cap<2**64 and pad>=0:cases += [(cap,pad,tiles,rows,bm),(cap,pad,max(0,tiles-1),rows,bm)] +cases += [(4096,4224,33,30,64),(4096,4224,33,4096,128),(4096,4224,33,4096,64)] +rng=random.Random(710) +for _ in range(1000):cases.append(tuple(rng.randrange(8193) for _ in range(5))) +for args in cases: + capacity,padded,tiles,rows,bm=args + want=int(0