Sync GPU paths to omega d946171 (#984 C6, #985, #986)

Makefile

@@ -621,17 +621,17 @@ test-gpu-imadhi-rc: $(GPU_IMADHI_TESTS)
 
 # omega-mv#554: static scoreboard hazard checker (RAW / WAR / WAW) over the encoded bytes of every kernel the generators
 # build, after the launchers' patch steps. Host only: decoder vs encoder, the pre-#553 attend red receipt
-# (tests/fixtures/bw_hazard), negative controls, then the sweep. test-bw-hazard-nvdisasm adds the decoder cross-check
-# against nvdisasm (offline oracle, /usr/local/cuda/bin) on every distinct kernel: one nvdisasm call per kernel, over
-# ten minutes on the Spark (2026-10-19: 2360 kernels, 1028064 instructions, 0 mismatched).
+# (tests/fixtures/bw_hazard), negative controls, then the sweep. test-bw-hazard-nvdisasm (the name it had when it
+# called nvdisasm) adds the decoder cross-check against Omega's own sm_121 disassembler, in process, on every distinct
+# kernel (bar C6: every check that called nvdisasm calls ours; test-sm121dis checks that disassembler against the oracle).
 .PHONY: test-bw-hazard test-bw-hazard-nvdisasm
 BW_HAZARD_TEST = $(OUT_DIR)/bw_hazard_test
-$(BW_HAZARD_TEST): tests/bw_hazard_test.c src/omega_bw_hazard.c src/omega_bw_hazard.h $(OUT_DIR)/libomega_gpu.a
+$(BW_HAZARD_TEST): tests/bw_hazard_test.c src/omega_bw_hazard.c src/omega_bw_hazard.h src/omega_sm121_disasm.h $(OUT_DIR)/libomega_gpu.a
 	$(CC) $(CFLAGS) -o $@ tests/bw_hazard_test.c src/omega_bw_hazard.c $(OUT_DIR)/libomega_gpu.a -lpthread -lm
 test-bw-hazard: $(BW_HAZARD_TEST)
 	$(BW_HAZARD_TEST)
 test-bw-hazard-nvdisasm: $(BW_HAZARD_TEST)
-	$(BW_HAZARD_TEST) --nvdisasm
+	$(BW_HAZARD_TEST) --disasm
 
 # Offline resource-DAG evaluator, imported verbatim from closed PR #590
 # (commit 3bd56731). Host arithmetic on the numbers written in the fixtures.

mk/sm121dis.mk

@@ -19,17 +19,22 @@ sm121dis-corpus-bins: $(addprefix $(OUT_DIR)/,gpu_matmul_api_test gpu_elementwis
 	gpu_moe_window_test gpu_fp8_api_test gpu_decode_attn_test gpu_router_test gpu_bf16_gemv_test gpu_fp8gemm_test \
 	gpu_tc_probe_test gpu_tc_peak_test gpu_pf_mem_test gpu_qmma_probe_test gpu_cmdlist_test gpu_decode_graph_test gpu_tcgemm_test \
 	gpu_prefill_attn_test gpu_pfgemm_test gpu_qgemm_test gpu_pfhost_test gpu_gpr_boundary_test \
-	gpu_scan_test gpu_gemm_test gpu_streams_test gpu_isolation_test)
+	gpu_scan_test gpu_gemm_test gpu_streams_test gpu_isolation_test bw_hazard_test)
 
-.PHONY: test-sm121dis sm121dis-corpus-bins test-sm121dis-corpus-controls
+.PHONY: test-sm121dis sm121dis-corpus-bins test-sm121dis-corpus-controls test-sm121dis-check-controls
 test-sm121dis-corpus-controls:
 	sh tools/sm121dis/corpus_selftest.sh
 
-test-sm121dis: $(SM121DIS) $(SM121DIS_RED) sm121dis-corpus-bins test-sm121dis-corpus-controls
+test-sm121dis-check-controls:
+	sh tools/sm121dis/check_selftest.sh
+
+test-sm121dis: $(SM121DIS) $(SM121DIS_RED) sm121dis-corpus-bins test-sm121dis-corpus-controls test-sm121dis-check-controls
 	rm -rf $(SM121DIS_K) && sh tools/sm121dis/corpus.sh $(OUT_DIR) $(SM121DIS_K)
 	sh tools/sm121dis/check.sh $(SM121DIS) $(SM121DIS_K) $(OUT_DIR)/sm121dis-work
 	@if sh tools/sm121dis/check.sh $(SM121DIS_RED) $(SM121DIS_K) $(OUT_DIR)/sm121dis-red > $(OUT_DIR)/sm121dis-red.log 2>&1; then \
 		echo "RED CHECK BROKEN: the faulty decoder passed"; exit 1; \
+	elif ! grep -q '^VERDICT FAIL' $(OUT_DIR)/sm121dis-red.log; then \
+		echo "RED CHECK BROKEN: the faulty decoder's run did not reach a verdict"; tail -3 $(OUT_DIR)/sm121dis-red.log; exit 1; \
 	else echo "red check: the faulty decoder FAILS the comparison, as it must ($$(grep -c '^\[FAIL\]' $(OUT_DIR)/sm121dis-red.log) kernels)"; fi
 
 # The provenance checks (tools/divsqrt_nvdisasm_check.sh, tools/ldst_nvdisasm_check.sh, test-numeric-reduce-nvdisasm)
@@ -44,7 +49,7 @@ endif
 # provenance check with a decoy nvdisasm first on PATH, the CUDA directories removed from PATH, and strace on execve;
 # then its own red check plants two calls to nvdisasm and must see the gate FAIL.
 .PHONY: test-zero-nvdisasm
-test-zero-nvdisasm: $(SM121DIS) build/sm121dis $(addprefix $(OUT_DIR)/,gpu_attention_test gpu_elementwise_test gpu_fp8_api_test gpu_reconv_test)
+test-zero-nvdisasm: $(SM121DIS) build/sm121dis $(addprefix $(OUT_DIR)/,gpu_attention_test gpu_elementwise_test gpu_fp8_api_test gpu_reconv_test bw_hazard_test)
 	PHYSICS_DIR="$(PHYSICS_DIR)" sh tools/sm121dis/zero_nvdisasm.sh $(OUT_DIR)
 	@if PHYSICS_DIR="$(PHYSICS_DIR)" sh tools/sm121dis/zero_nvdisasm.sh $(OUT_DIR) --red > $(OUT_DIR)/zero-nvdisasm-red.log 2>&1; then \
 		echo "RED CHECK BROKEN: the planted nvdisasm calls passed the gate"; exit 1; \

src/omega_blackwell_codegen.c

@@ -1143,9 +1143,12 @@ void omega_kernel_dump(const uint8_t *code, size_t len) {
     int fd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW | O_CLOEXEC, 0600);
     if (fd < 0) return;
     FILE *f = fdopen(fd, "wb");
-    if (!f) { close(fd); return; }
-    if (fwrite(code, 1, len, f) != len) BW_DIAG("omega_kernel_dump: short write %s\n", path);
-    if (fclose(f) != 0) BW_DIAG("omega_kernel_dump: close failed %s\n", path);
+    if (!f) { close(fd); unlink(path); BW_DIAG("omega_kernel_dump: write failed, removed %s\n", path); return; }
+    /* A body cut short (a full disk) must not stay under its full-content name: it would be compared as a smaller kernel
+     * and, created O_EXCL, never rewritten. tools/sm121dis/corpus.sh fails on these diagnostics. */
+    int bad = fwrite(code, 1, len, f) != len;
+    if (fclose(f) != 0) bad = 1;
+    if (bad) { unlink(path); BW_DIAG("omega_kernel_dump: write failed, removed %s\n", path); }
 #else
     (void)code; (void)len;
 #endif

src/omega_blackwell_codegen_tern.h

@@ -32,7 +32,8 @@
  * per word to an int32 total T; a SHFL.DOWN tree (8, 4, 2, 1) sums its half; y = T - KSC * 2^18 (mod 2^32, exact
  * because |y| <= 128 * k < 2^31).
  *
- * Shared memory: the launch descriptor allows at most OMEGA_BW_QMD_MAX_SHARED_BYTES (16 KiB) per CTA, so x is staged
+ * Shared memory: the launch descriptor allows at most OMEGA_BW_QMD_MAX_SHARED_BYTES (32 KiB, omega_blackwell_qmd.h) per CTA.
+ * x takes 2 bytes per column in shared memory, so one chunk of OMEGA_TERN_CHUNK_MAX columns uses 16 KiB, and x is staged
  * in chunks of at most OMEGA_TERN_CHUNK_MAX = 8192 columns (k <= 8192: one chunk; 8192 < k <= 16384: two equal chunks,
  * k % 512 == 0). With two chunks each lane l == 0 adds the earlier chunk's int32 y it stored itself (same thread, same
  * address, program order); the f32 epilogue is refused there.

src/omega_sm121_disasm.c

@@ -197,13 +197,21 @@ static const Row ROWS[] = {
     { 0x000000000000723eull, 0x00000010ffull, "F2FP.BF16.F32.PACK_AB %d, %a, %b" },
     /* tensor cores */
     { 0x000000000000723cull, 0x00041800ull, "HMMA.16816.F32.BF16 %d, %a, %b, %c" },
+    { 0x000000000000723cull, 0x00001800ull, "HMMA.16816.F32 %d, %a, %b, %c" },
     { 0x000000000000727aull, 0x00002c00ull, "QMMA.16832.F32.E4M3.E4M3 %d, %a, %b, %c" },
     { 0x0000000000007308ull, 0x00000800ull, "MUFU.EX2 %d, %b" },
     { 0x0000000000007308ull, 0x00001000ull, "MUFU.RCP %d, %b" },
     { 0x0000000000007308ull, 0x00001400ull, "MUFU.RSQ %d, %b" },
     /* memory */
     { 0x0000000000007981ull, 0x0c1e1100ull, "LDG.E%z %d, desc[%B][%a.64%o]" },
     { 0x0000000000007986ull, 0x0c101100ull, "STG.E%z desc[%C][%a.64%o], %b" },
+    /* the .STRONG.SYS forms (and ATOMG) are the ones the bw_hazard_test sweep builds (decode_attn); only the 32-bit
+     * size and the PT predicate seen there are rows, every other bit pattern stays reported, not guessed. The sweep
+     * has these forms only with a zero offset, so their %o is checked by the plain LDG/STG rows above (same opcode,
+     * same offset field, nonzero offsets throughout the corpus), not by a STRONG listing of its own */
+    { 0x0000000000007981ull, 0x0c1f5900ull, "LDG.E.STRONG.SYS %d, desc[%B][%a.64%o]" },
+    { 0x0000000000007986ull, 0x0c115900ull, "STG.E.STRONG.SYS desc[%C][%a.64%o], %b" },
+    { 0x80000000000079a8ull, 0x081f5100ull, "ATOMG.E.ADD.STRONG.SYS PT, %d, desc[%C][%a.64], %b" },
     { 0x0000000000007984ull, 0x08000800ull, "LDS %d, [%a+%B]" },
     { 0x0000000000007988ull, 0x08000800ull, "STS [%a+%C], %b" },
     { 0x0000000000007984ull, 0x00000000ull, "LDS%z %d, [%a%o]" },
@@ -229,10 +237,10 @@ static int imad_imm(Out *w, uint64_t lo, uint64_t hi) {
     if (imm == 0) name = "IMAD.MOV";                                  /* d = c */
     else if (imm == 1) name = c == 255 ? "IMAD.MOV" : "IMAD.IADD";    /* d = a, d = a + c */
     else if (c == 255 && (imm & (imm - 1)) == 0) {
-        /* a shift. Seen in the corpus: .SHL for 2^1..2^6, 2^9, 2^10, 2^12, 2^20; plain IMAD for 2^16 and 2^31 */
+        /* a shift. Seen in the corpus: .SHL for 2^1..2^10, 2^12, 2^20; plain IMAD for 2^16 and 2^31 */
         int n = __builtin_ctz(imm);
         if (n == 16 || n == 31) name = "IMAD";
-        else if (n <= 6 || n == 9 || n == 10 || n == 12 || n == 20) name = "IMAD.SHL";
+        else if (n <= 10 || n == 12 || n == 20) name = "IMAD.SHL";
         else return -1;
     }
     put(w, "%s ", name);

tests/bw_hazard_test.c

@@ -10,9 +10,12 @@
  *   4. sweep: every generator over its accepted shapes (production kernels, mutant 0), encoded and patched as the
  *      launchers do; every finding is printed and must be on the triaged list below.
  *
- * --nvdisasm: additionally cross-checks the decoder against nvdisasm (offline oracle, CUDA 13 -b SM121) on every
- * distinct kernel: the base registers each instruction names must be the ones the decoder found. */
+ * --disasm: additionally cross-checks the decoder against Omega's own sm_121 disassembler (src/omega_sm121_disasm.c,
+ * in process) on every distinct kernel: the base registers each instruction names must be the ones the decoder found.
+ * The disassembler is checked word for word against nvdisasm by test-sm121dis; this test never calls nvdisasm (bar C6
+ * of docs/OMEGA.md: every check that called nvdisasm calls ours). */
 #include "omega_bw_hazard.h"
+#include "omega_sm121_disasm.h"
 #include "omega_blackwell_codegen.h"
 #include "omega_blackwell_codegen_moe.h"
 #include "omega_blackwell_codegen_moe2.h"
@@ -38,7 +41,7 @@
 #include <string.h>
 #include <unistd.h>
 
-static int g_checks, g_failed, g_nvdisasm;
+static int g_checks, g_failed, g_disasm;
 static const char *g_dump_dir; /* --dump DIR: write each distinct kernel with findings as DIR/<family>-<n>.bin (triage) */
 #define CHECK(c, ...) do { g_checks++; if (!(c)) { g_failed++; printf("FAIL: "); printf(__VA_ARGS__); printf("\n"); } } while (0)
 
@@ -348,7 +351,7 @@ static Known g_known[] = {
 
 static uint64_t fnv(const uint8_t *p, size_t n) { uint64_t h = 1469598103934665603ull; for (size_t i = 0; i < n; i++) { h ^= p[i]; h *= 1099511628211ull; } return h; }
 
-static void nvdisasm_cross_check(const char *family, const char *variant, const uint8_t *code, size_t len);
+static void disasm_cross_check(const char *family, const char *variant, const uint8_t *code, size_t len);
 
 static void sweep_bytes(const char *family, const char *variant, const uint8_t *code, size_t len) {
     g_variants++;
@@ -381,7 +384,7 @@ static void sweep_bytes(const char *family, const char *variant, const uint8_t *
         if (fd >= 0 && !f) close(fd);
         if (f) { fwrite(code, 1, len, f); fclose(f); printf("DUMP %s = %s %s\n", path, family, variant); }
     }
-    if (g_nvdisasm) nvdisasm_cross_check(family, variant, code, len);
+    if (g_disasm) disasm_cross_check(family, variant, code, len);
 }
 
 /* encode + the launcher's patch steps */
@@ -563,30 +566,25 @@ static void sweep(void) {
       if (omega_bw_ir_append(&g_prog, &ex) >= 0 && omega_bw_ir_append(&g_prog, &self) >= 0 && omega_bw_regalloc_solve(&g_prog) == 0) sweep_prog("cmdlist_nop", "-", P_NONE); }
 }
 
-/* ----------------------------------------------------------------------------------------- nvdisasm cross-check */
-static size_t g_nv_insns, g_nv_bad, g_nv_kernels;
+/* ----------------------------------------------------------------------------------- disassembler cross-check (C6) */
+static size_t g_dis_insns, g_dis_bad, g_dis_kernels;
 static int decoded_has(const OmegaBwInsn *d, int reg, int uniform) {
     for (unsigned j = 0; j < d->nr; j++) if (d->rd[j].uniform == uniform && d->rd[j].reg == reg) return 1;
     for (unsigned j = 0; j < d->nw; j++) if (d->wr[j].uniform == uniform && d->wr[j].reg == reg) return 1;
     return 0;
 }
-static void nvdisasm_cross_check(const char *family, const char *variant, const uint8_t *code, size_t len) {
-    char path[] = "/tmp/bw_hazard_nvdisasm_XXXXXX";
-    int fd = mkstemp(path); if (fd < 0) { CHECK(0, "mkstemp"); return; }
-    FILE *f = fdopen(fd, "wb"); fwrite(code, 1, len, f); fclose(f);
-    char cmd[600]; snprintf(cmd, sizeof cmd, "/usr/local/cuda/bin/nvdisasm -b SM121 -raw '%s' 2>&1", path);
-    FILE *p = popen(cmd, "r");
-    if (!p) { remove(path); CHECK(0, "nvdisasm popen"); return; }
-    char line[1024]; size_t seen = 0; g_nv_kernels++;
-    while (fgets(line, sizeof line, p)) {
-        char *s = strstr(line, "/*"); if (!s || s[6] != '*' || s[7] != '/') continue; /* the address field, e.g. 0040 */
-        unsigned addr; if (sscanf(s + 2, "%4x", &addr) != 1) continue;
-        size_t pc = addr / 16u; if (pc * 16u >= len) continue;
-        char *t = s + 8; while (*t == ' ') t++;
-        char *end = strchr(t, ';'); if (end) *end = 0;
-        OmegaBwInsn d; if (omega_bw_decode(code + pc * 16u, &d) != 0) continue; /* unreachable padding words are not decoded either */
-        seen++; g_nv_insns++;
-        /* every register token nvdisasm prints must be a base register the decoder has, and vice versa */
+static void disasm_cross_check(const char *family, const char *variant, const uint8_t *code, size_t len) {
+    size_t seen = 0; g_dis_kernels++;
+    for (size_t pc = 0; pc * 16u + 16u <= len; pc++) {
+        const uint8_t *w = code + pc * 16u;
+        OmegaBwInsn d; if (omega_bw_decode(w, &d) != 0) continue; /* unreachable padding words are not decoded either */
+        seen++; g_dis_insns++;
+        uint64_t lo = 0, hi = 0;
+        for (int i = 7; i >= 0; i--) { lo = lo << 8 | w[i]; hi = hi << 8 | w[8 + i]; }
+        char t[256];
+        int rc = omega_sm121_disasm(lo, hi, pc * 16u, t, sizeof t);
+        if (rc != 0) { g_dis_bad++; if (g_dis_bad <= 20) printf("DISASM UNKNOWN %s %s IR %zu: decoder %s, disassembler rc %d lo %016llx hi %016llx\n", family, variant, pc, d.name, rc, (unsigned long long)lo, (unsigned long long)hi); continue; }
+        /* every register token the disassembler prints must be a base register the decoder has, and vice versa */
         int nvr[16], nvu[16], nn = 0, nu = 0;
         for (char *q = t; *q; q++) {
             int uni = 0; char *r = q;
@@ -605,16 +603,15 @@ static void nvdisasm_cross_check(const char *family, const char *variant, const
             for (int i = 0; i < (o->uniform ? nu : nn); i++) if ((o->uniform ? nvu[i] : nvr[i]) == o->reg) f2 = 1;
             ok = f2;
         }
-        if (!ok) { g_nv_bad++; if (g_nv_bad <= 20) printf("NVDISASM MISMATCH %s %s IR %zu: '%s' vs decoder %s (%u reads, %u writes)\n", family, variant, pc, t, d.name, d.nr, d.nw); }
+        if (!ok) { g_dis_bad++; if (g_dis_bad <= 20) printf("DISASM MISMATCH %s %s IR %zu: '%s' vs decoder %s (%u reads, %u writes)\n", family, variant, pc, t, d.name, d.nr, d.nw); }
     }
-    int st = pclose(p); remove(path);
-    CHECK(st == 0 && seen > 0, "%s %s: nvdisasm ran (status %d, %zu instructions)", family, variant, st, seen);
+    CHECK(seen > 0, "%s %s: disassembled %zu instructions", family, variant, seen);
 }
 
 int main(int argc, char **argv) {
     const char *dir = "tests/fixtures/bw_hazard";
     for (int i = 1; i < argc; i++) {
-        if (!strcmp(argv[i], "--nvdisasm")) g_nvdisasm = 1;
+        if (!strcmp(argv[i], "--disasm")) g_disasm = 1;
         else if (!strcmp(argv[i], "--fixtures") && i + 1 < argc) dir = argv[++i];
         else if (!strcmp(argv[i], "--dump") && i + 1 < argc) g_dump_dir = argv[++i];
     }
@@ -640,9 +637,9 @@ int main(int argc, char **argv) {
     printf("sweep: %zu variants built, %zu distinct kernels, %zu clean, %zu with findings, %zu findings, %zu not checked\n",
            g_variants, g_nseen, clean, g_nseen - clean - g_unchecked, g_findings_total, g_unchecked);
     CHECK(g_nseen >= 50, "the sweep built only %zu distinct kernels", g_nseen);
-    if (g_nvdisasm) {
-        printf("nvdisasm cross-check: %zu kernels, %zu instructions, %zu mismatched\n", g_nv_kernels, g_nv_insns, g_nv_bad);
-        CHECK(g_nv_bad == 0 && g_nv_insns > 0, "nvdisasm cross-check");
+    if (g_disasm) {
+        printf("disassembler cross-check (sm121dis, in process): %zu kernels, %zu instructions, %zu mismatched\n", g_dis_kernels, g_dis_insns, g_dis_bad);
+        CHECK(g_dis_bad == 0 && g_dis_insns > 0, "disassembler cross-check");
     }
     printf("bw_hazard_test: %d checks, %d failed: %s\n", g_checks, g_failed, g_failed ? "FAIL" : "PASS");
     return g_failed ? 1 : 0;

tools/extract/.crumb

@@ -3,14 +3,14 @@
     "generated": {
       "children": [],
       "children_root": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-      "compiled_at": "2026-10-31T02:18:53Z",
+      "compiled_at": "2026-10-31T03:41:35Z",
       "compiler": "crumb-compile/1",
-      "digest": "sha256:a35550169b7ad8d28e821d6e5354a9a3ced03dcf9556dbf5c96dd08310c4944a",
+      "digest": "sha256:018d5c842d33624b6f35f824c1f325691028b0fe832d0ef5c89eb6107e8532e7",
       "evidence_files": 0,
       "evidence_root": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
       "files": 30,
-      "generated_at_commit": "02538d3bd12a14fce0c5875ac491203919e505ef",
-      "source_tree": "sha256:396ddda55052497750b75bebda7820fb6f86c309fb153a738b2744c65b74a94b"
+      "generated_at_commit": "f8abf4041b2f6ad69c971a9d493d93e868eb57df",
+      "source_tree": "sha256:33f126b6ea484c161022ed00a9a8d8c0be3a427597751b29290518aac5fb4761"
     }
   },
   "name": "extract",