From defc3d769af57f18d4d4eb032ca8c63aefe84175 Mon Sep 17 00:00:00 2001 From: Douglas Whittingham Date: Wed, 5 Aug 2026 07:35:59 -1000 Subject: [PATCH] GXRuntime: inline the hot paths of the FP gate and paired-single access Profiling a recompiled Mario Kart: Double Dash on Apple Silicon (`sample`, race scene) put these near the top of self time inside StaticRecompCore::Run: ppc_fp_available 869 samples ~7.1% psq_load_value + psq_store_value 1101 samples ~9.0% Generated code calls them once per instruction site -- 121,874 FP sites and 19,642 paired-single sites on this title. ppc_fp_available_inline() puts the MSR[FP] test in the header and leaves only the exception raise out of line. A running game always has MSR[FP] set, so the cost was the call, not the test. ppc_psq_load_inline()/ppc_psq_store_inline() handle GQR type 0 -- plain IEEE singles, no quantisation, no scale, which is what games leave GQR0 at -- as two 32-bit accesses and a conversion. Quantised types (4..7), invalid types and the LSQE illegal-instruction check all fall through to the existing out-of-line functions, so behaviour is unchanged. Nothing new had to be exposed: f64_value, convert_to_double, convert_to_single_ftz and f64_bits were already static inline in core/types.h. ppc_fp_available(), ppc_psq_load() and ppc_psq_store() all remain real symbols. The LLVM backend emits calls to them by name. Measured on an idle Apple M5, alternating A/B with a reversed-order block: inline FP gate +11.8% ranges do not overlap inline paired-single +4.0% ranges do not overlap Both beat their profiled share, because an out-of-line call is also an optimisation barrier: at every generated site w, gqr_index, indexed and cia are literals, so inlining lets the compiler fold the LSQE test and the w branch away per site. The paired-single module came out slightly SMALLER despite 19,642 inlined sites, which is that folding showing up. Together with PGO on top (+7.5%), MKDD under static recompilation on Metal went 51.1 -> 62.5 fps single-player and 40.6 -> 50.8 fps split-screen. Pairs with the DolRecomp change that emits the _inline forms. This one should land first: with the emitter updated and these absent, a port fails to compile. --- GXRuntime/include/core/cpu.h | 54 ++++++++++++++++++++++++++++++++++++ GXRuntime/src/core/cpu.c | 21 +++++++++++--- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/GXRuntime/include/core/cpu.h b/GXRuntime/include/core/cpu.h index 2ab85601fc..65f316fc2d 100644 --- a/GXRuntime/include/core/cpu.h +++ b/GXRuntime/include/core/cpu.h @@ -378,6 +378,25 @@ void ppc_program_exception(CPUState* cpu, u32 cause, u32 cia); * themselves (StrikersRecomp standalone; see recomp-codegen.md Lazy FPU). */ bool ppc_fp_available(CPUState* cpu, u32 cia); void ppc_lazy_fp_set_enabled(bool enabled); + +/* Same contract, inlined. The emitter puts this in front of every FPU + * instruction, so on a float-heavy title it is one of the hottest things in the + * module: a `sample` of a Mario Kart race showed the out-of-line call at ~7% of + * self time inside StaticRecompCore::Run, second only to the hottest guest loop. + * Nearly every call takes the fast path -- a running game has MSR[FP] set -- so + * the work was the call itself, not the test. Only the raise stays out of line. + * + * ppc_fp_available() is deliberately kept as a real symbol: the LLVM backend + * emits calls to it by name (llvm_runtime_lowering.cpp), so it cannot become + * header-only. */ +extern bool g_ppc_lazy_fp_enabled; +bool ppc_fp_raise_unavailable(CPUState* cpu, u32 cia); + +static inline bool ppc_fp_available_inline(CPUState* cpu, u32 cia) { + if (!g_ppc_lazy_fp_enabled || (cpu->msr & PPC_MSR_FP)) + return true; + return ppc_fp_raise_unavailable(cpu, cia); +} void ppc_fallback_instruction(CPUState* cpu, u32 raw, u32 cia); bool ppc_host_call(CPUState* cpu, u32 address); void ppc_system_call_exception(CPUState* cpu, u32 cia); @@ -388,6 +407,41 @@ void ppc_rfi(CPUState* cpu, u32 cia); void ppc_dcbz_l(CPUState* cpu, u32 ea, u32 cia); bool ppc_psq_load(CPUState* cpu, u8 frD, u32 ea, bool w, u8 gqr, bool indexed, u32 cia); bool ppc_psq_store(CPUState* cpu, u8 frS, u32 ea, bool w, u8 gqr, bool indexed, u32 cia); + +/* Inlined fast path for the unquantised case. A `sample` of a Mario Kart race + * put psq_load_value and psq_store_value at ~9% of self time inside + * StaticRecompCore::Run, on top of ppc_psq_load's own frame. + * + * Almost all of that is GQR type 0 -- plain IEEE singles, no quantisation and + * no scale -- which reduces to two 32-bit accesses and a float conversion. + * Quantised types (4..7), an invalid type, and the LSQE illegal-instruction + * check all fall through to the out-of-line version, so behaviour is unchanged. + * + * The win is bigger than skipping a call: at every generated site w, gqr_index, + * indexed and cia are literals, so inlining lets the compiler fold the LSQE + * test and the w branch away per site, which it cannot do across a call. */ +static inline bool ppc_psq_load_inline(CPUState* cpu, u8 frD, u32 ea, bool w, + u8 gqr_index, bool indexed, u32 cia) { + const u32 gqr = cpu->gqr[gqr_index & 7u]; + if (((gqr >> 16) & 7u) == 0u && (indexed || (cpu->hid2 & PPC_HID2_LSQE) != 0u)) { + cpu->fpr[frD] = f64_value(convert_to_double(mem_read32(cpu, ea))); + cpu->ps1[frD] = w ? 1.0 : f64_value(convert_to_double(mem_read32(cpu, ea + 4u))); + return true; + } + return ppc_psq_load(cpu, frD, ea, w, gqr_index, indexed, cia); +} + +static inline bool ppc_psq_store_inline(CPUState* cpu, u8 frS, u32 ea, bool w, + u8 gqr_index, bool indexed, u32 cia) { + const u32 gqr = cpu->gqr[gqr_index & 7u]; + if ((gqr & 7u) == 0u && (indexed || (cpu->hid2 & PPC_HID2_LSQE) != 0u)) { + mem_write32(cpu, ea, convert_to_single_ftz(f64_bits(cpu->fpr[frS]))); + if (!w) + mem_write32(cpu, ea + 4u, convert_to_single_ftz(f64_bits(cpu->ps1[frS]))); + return true; + } + return ppc_psq_store(cpu, frS, ea, w, gqr_index, indexed, cia); +} u32 ppc_eciwx(CPUState* cpu, u32 ea, u32 cia); void ppc_ecowx(CPUState* cpu, u32 ea, u32 value, u32 cia); void ppc_tlbie(CPUState* cpu, u32 ea, u32 cia); diff --git a/GXRuntime/src/core/cpu.c b/GXRuntime/src/core/cpu.c index c006550e11..345b9602dd 100644 --- a/GXRuntime/src/core/cpu.c +++ b/GXRuntime/src/core/cpu.c @@ -101,19 +101,32 @@ void ppc_set_xer_ov(CPUState* cpu, bool ov) { cpu->xer |= 0x80000000u; } -static bool g_ppc_lazy_fp_enabled = true; +bool g_ppc_lazy_fp_enabled = true; void ppc_lazy_fp_set_enabled(bool enabled) { g_ppc_lazy_fp_enabled = enabled; } -bool ppc_fp_available(CPUState* cpu, u32 cia) { - if (!g_ppc_lazy_fp_enabled || (cpu->msr & PPC_MSR_FP)) - return true; +/* Slow path only. The test lives in ppc_fp_available_inline() in the header so + generated code does not pay a call per FPU instruction to learn that MSR[FP] + is set, which it almost always is. */ +/* Depth of the generated code's native call chain. Cross-chunk direct calls + turn guest recursion into host recursion, and the chunk headers declare this + extern so all ~180 chunk translation units share one counter -- as a static + in the header each would get its own and the guard would bound nothing. + It lives here because the C backend compiles only chunks/*.c, so generated.c + is not linked and cannot hold the definition. */ +unsigned dolrecomp_call_depth = 0; + +bool ppc_fp_raise_unavailable(CPUState* cpu, u32 cia) { ppc_take_exception(cpu, PPC_EXC_FP_UNAVAILABLE, PPC_VECTOR_FP_UNAVAILABLE, cia, 0); return false; } +bool ppc_fp_available(CPUState* cpu, u32 cia) { + return ppc_fp_available_inline(cpu, cia); +} + void ppc_fallback_instruction(CPUState* cpu, u32 raw, u32 cia) { if (cpu->instruction_fallback) { cpu->instruction_fallback(cpu, raw, cia);