From 78c24992d06f27955a723215bc1c90d9aff27d45 Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Wed, 7 Oct 2026 20:16:48 -0700 Subject: [PATCH 01/13] feat(emulator): add apple m1 --- README.md | 32 + src/emulator/cli/src/boot.rs | 25 +- src/emulator/src/aarch64/compile.rs | 11 +- src/emulator/src/aarch64/cpu.rs | 11 + src/emulator/src/aarch64/decode.rs | 67 +- src/emulator/src/aarch64/exception.rs | 8 +- src/emulator/src/aarch64/identification.rs | 12 +- src/emulator/src/aarch64/translate.rs | 47 +- src/emulator/src/devices/aic.rs | 470 ++++++++++++ src/emulator/src/devices/mod.rs | 3 + src/emulator/src/devices/pmgr.rs | 93 +++ src/emulator/src/devices/s5l_uart.rs | 358 ++++++++++ src/emulator/src/system/boot.rs | 32 + src/emulator/src/system/dtb.rs | 794 +++++++++++++++++++++ src/emulator/src/system/m1.rs | 216 ++++++ src/emulator/src/system/machine.rs | 119 ++- src/emulator/src/system/mod.rs | 247 +++++-- 17 files changed, 2402 insertions(+), 143 deletions(-) create mode 100644 src/emulator/src/devices/aic.rs create mode 100644 src/emulator/src/devices/pmgr.rs create mode 100644 src/emulator/src/devices/s5l_uart.rs create mode 100644 src/emulator/src/system/dtb.rs create mode 100644 src/emulator/src/system/m1.rs diff --git a/README.md b/README.md index 7bab141..3399e49 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,38 @@ enforced by translation, the physical timer is stored but never fires, floating rounds to nearest even and sets no `FPSR` flags, and the GICv3 carries only the virtual timer (no SPIs, SGIs or group 1). +## Apple M1 (t8103) SoC + +`--dtb` selects the M1 machine in place of the QEMU `virt` board. The device tree is +the Asahi kernel's board tree, which the kernel tree builds as +`arch/arm64/boot/dts/apple/t8103-j274.dtb`: + +```sh +cargo run --release --bin volt-boot -- \ + ~/src/linux/arch/arm64/boot/Image 60 \ + --dtb ~/src/linux/arch/arm64/boot/dts/apple/t8103-j274.dtb +``` + +The SoC addresses come from the M1 Mac mini's own device tree (`DeviceTree.j274ap` +in a macOS restore image; `ipsw dtree` reads it). DRAM is at `0x8_0000_0000`, the +interrupt controller (AIC) at `0x2_3b10_0000`, the power-state blocks at +`0x2_3b70_0000` and `0x2_3d28_0000`, and the console (`apple,s5l-uart`, `stdout-path`) +at `0x2_3520_0000`. The machine is one vCPU with the M1 Icestorm identity +(`MIDR_EL1` 0x610f0220) and a 16 KiB granule, which the Asahi kernel requires. + +The Linux kernel boots through the AIC and the S5L console, probes its power domains, +and panics on the missing root filesystem (`VFS: Unable to mount root fs`), which is +the expected result without one. `--initrd`, `--disk` and `--smp` above 1 are rejected +on this board, and a Mach-O kernel is refused: XNU needs the SoC's own ADT and +firmware interfaces, which are not modelled. + +Not modelled: any arm-io device other than the AIC, the UART and the power-state +blocks. Those addresses read as zero and writes are dropped, and the boot report counts +them as `unmodeled device accesses`, so driver probes of missing hardware (DART, GPU, +SMC, PCIe) fail rather than fault. Secondary cores stay powered off (the device tree +gives them no release address), the physical timer never fires, and fast IPIs are +storage only. + ## Performance Translating a block costs much more than running it once, and a kernel boot runs most diff --git a/src/emulator/cli/src/boot.rs b/src/emulator/cli/src/boot.rs index 8832cc9..9ccd3cd 100644 --- a/src/emulator/cli/src/boot.rs +++ b/src/emulator/cli/src/boot.rs @@ -8,8 +8,8 @@ use std::{ }; use volt_emulator::devices::BlockBackend; use volt_emulator::system::{ - DEFAULT_CMDLINE, DEFAULT_XNU_CMDLINE, IdleMode, RAM_SIZE, SystemConfig, XNU_RAM_SIZE, - smp::SmpConfig, + Board, DEFAULT_CMDLINE, DEFAULT_XNU_CMDLINE, IdleMode, RAM_SIZE, SystemConfig, XNU_RAM_SIZE, + m1, smp::SmpConfig, }; use winnow_args::{Args, ValueEnum}; @@ -36,7 +36,7 @@ impl From for IdleMode { #[derive(Args, Debug)] #[arg( name = "volt-boot", - about = "AArch64 Linux or XNU boot with PL011, GICv2/GICv3, and VirtIO-Blk (Linux)" + about = "AArch64 Linux or XNU boot with PL011, GICv2/GICv3, VirtIO-Blk, or the Apple M1 SoC" )] struct Cli { /// Path to a raw Linux AArch64 Image, or an XNU kernel (Mach-O). @@ -59,6 +59,11 @@ struct Cli { #[arg(short, long, alias = "initramfs")] initrd: Option, + /// Device tree for the Apple M1 (t8103) SoC, such as the Asahi kernel's + /// `t8103-j274.dtb`. Selects that machine in place of the QEMU `virt` board. + #[arg(long)] + dtb: Option, + /// Guest RAM size (e.g. 128M, 256M, 512M, 1G). #[arg(short, long, alias = "ram")] memory: Option, @@ -193,10 +198,18 @@ fn run(cli: Cli) -> Result<(), String> { let seconds = cli.seconds.unwrap_or(20); let image = std::fs::read(&cli.image).map_err(|e| format!("{}: {e}", cli.image.display()))?; + let board = match &cli.dtb { + Some(path) => Board::AppleM1 { + device_tree: std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?, + }, + None => Board::Virt, + }; // A Mach-O image is an XNU kernel; anything else is a raw Linux Image. let xnu = volt_emulator::system::xnu::is_macho(&image); let default_cmdline = if xnu { DEFAULT_XNU_CMDLINE + } else if matches!(board, Board::AppleM1 { .. }) { + m1::DEFAULT_CMDLINE } else if cli.disk.is_some() { "root=/dev/vda rw console=ttyAMA0 earlycon=pl011,0x9000000 nokaslr loglevel=8" } else { @@ -213,6 +226,8 @@ fn run(cli: Cli) -> Result<(), String> { RAM_SIZE.max((initrd.len() * 3 + (128 << 20)).next_power_of_two()) } else if xnu { XNU_RAM_SIZE + } else if matches!(board, Board::AppleM1 { .. }) { + m1::DEFAULT_RAM_SIZE } else { RAM_SIZE }; @@ -227,6 +242,9 @@ fn run(cli: Cli) -> Result<(), String> { None => None, }; + if cli.cpus > 1 && !matches!(board, Board::Virt) { + return Err("the Apple M1 SoC boots one vCPU; pass --smp 1".into()); + } if cli.cpus > 1 { let config = SmpConfig { cpus: cli.cpus, @@ -256,6 +274,7 @@ fn run(cli: Cli) -> Result<(), String> { ram_size, initrd: initrd_data, disk: disk_backend, + board, }; let mut output = std::io::BufWriter::new(std::io::stdout()); let report = volt_emulator::system::boot_system(&image, config, move |byte| { diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index b1365ad..21cab61 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -2147,7 +2147,10 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< | Instruction::Dmb | Instruction::Yield => {} Instruction::Wfe => l.trap(pc.wrapping_add(4), Trap::Wfe), - Instruction::Trap => l.trap(pc, Trap::Brk), + Instruction::Brk(imm) => { + l.store(offset_of!(Cpu, address), l.k(l.u64, u64::from(imm))); + l.trap(pc, Trap::Brk) + } Instruction::Isb => l.trap(pc.wrapping_add(4), Trap::Isb), Instruction::Tlbi => l.trap(pc.wrapping_add(4), Trap::Tlbi), Instruction::DcZva(rt) => { @@ -2464,6 +2467,10 @@ fn system_offset(r: SystemRegister) -> Option { Fpcr => offset_of!(CpuSystem, fpcr), Fpsr => offset_of!(CpuSystem, fpsr), Daif => offset_of!(CpuSystem, daif), + Apple(slot) => offset_of!(CpuSystem, apple) + 8 * usize::from(slot), + MidrEl1 => offset_of!(CpuSystem, midr_el1), + IdAa64pfr0El1 => offset_of!(CpuSystem, id_aa64pfr0_el1), + IdAa64mmfr0El1 => offset_of!(CpuSystem, id_aa64mmfr0_el1), _ => return None, }) } @@ -2531,7 +2538,7 @@ fn lower_system(l: &Lower, a: System) { l.store(pan, l.imm(l.u64, B::BitAnd, bit, 1)); } } - RazWi => { + RazWi | AppleZero => { if a.read { l.put(a.rt, l.k(l.u64, 0)); } diff --git a/src/emulator/src/aarch64/cpu.rs b/src/emulator/src/aarch64/cpu.rs index 8e37c64..a2d1b7b 100644 --- a/src/emulator/src/aarch64/cpu.rs +++ b/src/emulator/src/aarch64/cpu.rs @@ -199,6 +199,13 @@ pub struct System { pub icc_igrpen0_el1: u64, pub cntp_ctl_el0: u64, pub actlr_el1: u64, + /// Apple IMP-DEF configuration registers, indexed by `decode::SystemRegister::Apple`. + pub apple: [u64; crate::aarch64::decode::APPLE_SLOTS], + /// `MIDR_EL1`, `ID_AA64PFR0_EL1` and `ID_AA64MMFR0_EL1`: the identity this CPU + /// advertises, set by the board. + pub midr_el1: u64, + pub id_aa64pfr0_el1: u64, + pub id_aa64mmfr0_el1: u64, } impl Default for System { fn default() -> Self { @@ -243,6 +250,10 @@ impl Default for System { icc_igrpen0_el1: 0, cntp_ctl_el0: 0, actlr_el1: 0, + apple: [0; crate::aarch64::decode::APPLE_SLOTS], + midr_el1: crate::aarch64::identification::DEFAULT_MIDR_EL1, + id_aa64pfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64PFR0_EL1, + id_aa64mmfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64MMFR0_EL1, } } } diff --git a/src/emulator/src/aarch64/decode.rs b/src/emulator/src/aarch64/decode.rs index 8d10f55..1f445dc 100644 --- a/src/emulator/src/aarch64/decode.rs +++ b/src/emulator/src/aarch64/decode.rs @@ -601,6 +601,11 @@ pub enum SystemRegister { ActlrEl1, Nzcv, RazWi, + /// Read-as-zero, write-ignored: Apple IMP-DEF status registers (`op0` 3). + AppleZero, + /// Apple IMP-DEF configuration register the guest reads back, stored in + /// `System::apple[slot]`. + Apple(u8), } impl SystemRegister { pub fn read_only(self) -> bool { @@ -814,7 +819,7 @@ pub enum Instruction { Sevl, Wfe, Yield, - Trap, + Brk(u16), B(i64), BCond(BranchCondition), Csel(Select), @@ -894,7 +899,7 @@ impl Instruction { | Psci | Wfi | Wfe - | Trap + | Brk(_) | Isb | DcZva(_) | AddressTranslate(_) @@ -1081,6 +1086,48 @@ fn simd_struct(word: u32) -> Result, DecodeErro } Ok(Some((desc, post))) } +/// Number of Apple IMP-DEF configuration registers `System::apple` holds. +pub const APPLE_SLOTS: usize = 21; + +/// The `S3_*` implementation-defined registers the M1 kernel and PMU driver touch +/// at EL1 (`arch/arm64/include/asm/apple_m1_pmu.h`, `irq-apple-aic.c`). Configuration +/// is stored so the guest reads back what it wrote. Status registers (IPI pending, +/// PMU interrupt active, PMU state) read as zero: this model raises no fast IPIs and +/// no PMU interrupts. Anything else falls through to the unsupported path. +fn apple_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option { + use SystemRegister::*; + if op0 != 3 || crn != 15 { + return None; + } + Some(match (op1, crm, op2) { + (1, 0, 0) => Apple(0), + (1, 1, 0) => Apple(1), + (1, 2, 0) => Apple(2), + (1, 3, 0) => Apple(3), + (1, 4, 0) => Apple(4), + (1, 5, 0) => Apple(5), + (1, 6, 0) => Apple(6), + (1, 13, 0) => AppleZero, + (2, 0, 0) => Apple(7), + (2, 1, 0) => Apple(8), + (2, 2, 0) => Apple(9), + (2, 3, 0) => Apple(10), + (2, 4, 0) => Apple(11), + (2, 5, 0) => Apple(12), + (2, 6, 0) => Apple(13), + (2, 7, 0) => Apple(14), + (2, 9, 0) => Apple(15), + (2, 10, 0) => Apple(16), + (5, 0, 0) => Apple(17), + (5, 0, 1) => Apple(18), + (5, 1, 1) => AppleZero, + (5, 3, 1) => Apple(19), + (7, 0, 4) => Apple(20), + (7, 6, 4) => AppleZero, + _ => return None, + }) +} + fn system_register( op0: u32, op1: u32, @@ -1092,6 +1139,9 @@ fn system_register( if op0 == 2 && op1 == 0 && crn == 0 && (4..=7).contains(&op2) { return Ok(RazWi); } + if let Some(register) = apple_register(op0, op1, crn, crm, op2) { + return Ok(register); + } if op1 == 0 && crn == 4 && op2 == 5 { return Ok(SpSelSet); } @@ -1753,12 +1803,10 @@ pub fn decode(word: u32) -> Result { immediate: ((word >> 8) & 15) as u8, })); } - if word & 0xffe0_f000 == 0xd420_0000 { - return if word & 0x0000_ffe0 == 0x20 { - Ok(Trap) - } else { - Err(UnsupportedInstruction) - }; + // BRK #imm16: opc 001 with LL 00. Its immediate is the whole field, so any value + // decodes; the trap carries it to the exception syndrome. + if word & 0xffe0_001f == 0xd420_0000 { + return Ok(Brk((word >> 5) as u16)); } if word & 0x1f00_0000 == 0x1000_0000 { let page = word & 0x8000_0000 != 0; @@ -2493,7 +2541,8 @@ mod tests { (0xd69f03e0, Eret), (0xd4000001, Svc), (0xd4000002, Psci), - (0xd4200020, Trap), + (0xd4200020, Brk(1)), + (0xd4210000, Brk(0x800)), ] { assert_eq!(decode(word), Ok(expected)); } diff --git a/src/emulator/src/aarch64/exception.rs b/src/emulator/src/aarch64/exception.rs index b0c5260..bc9ad9c 100644 --- a/src/emulator/src/aarch64/exception.rs +++ b/src/emulator/src/aarch64/exception.rs @@ -15,6 +15,8 @@ pub struct Reason { pub ec: u8, pub instruction: bool, pub status: u8, + /// Further ISS bits: a BRK's immediate. Syndromes with a DFSC leave this 0. + pub iss: u32, } impl Default for Reason { fn default() -> Self { @@ -23,6 +25,7 @@ impl Default for Reason { ec: 0, instruction: false, status: status::TRANSLATION, + iss: 0, } } } @@ -30,6 +33,8 @@ pub mod status { pub const TRANSLATION: u8 = 0b000100; pub const PERMISSION: u8 = 0b001100; pub const PERMISSION_FETCH: u8 = 0b001101; + /// Synchronous external abort: the access reached no memory or device. + pub const SYNC_EXTERNAL: u8 = 0b010000; } pub fn take(cpu: &mut Cpu, reason: Reason, fault_address: u64) -> u64 { let from = cpu.system.el; @@ -41,7 +46,8 @@ pub fn take(cpu: &mut Cpu, reason: Reason, fault_address: u64) -> u64 { cpu.system.elr_el1 = cpu.pc; cpu.system.esr_el1 = (u64::from(reason.ec & 63) << 26) | (u64::from(reason.instruction) << 25) - | u64::from(reason.status & 63); + | u64::from(reason.status & 63) + | u64::from(reason.iss & 0x1ff_ffff); cpu.system.far_el1 = fault_address; let group = if from != 1 { 2 diff --git a/src/emulator/src/aarch64/identification.rs b/src/emulator/src/aarch64/identification.rs index 46a7ba2..595e76b 100644 --- a/src/emulator/src/aarch64/identification.rs +++ b/src/emulator/src/aarch64/identification.rs @@ -16,19 +16,21 @@ pub const CCSIDR_L1D: u64 = ccsidr(4, 128, 4); pub const CCSIDR_L1I: u64 = ccsidr(2, 128, 4); /// L2 unified: 1 MiB, 8-way. pub const CCSIDR_L2: u64 = ccsidr(4, 2048, 8); +/// The virt board's `MIDR_EL1`, `ID_AA64PFR0_EL1` and `ID_AA64MMFR0_EL1`. A board +/// may advertise different identity, so `value` leaves these three to the CPU state. +pub const DEFAULT_MIDR_EL1: u64 = 0x410f_d0f0; +pub const DEFAULT_ID_AA64PFR0_EL1: u64 = (0xf << 20) | (0xf << 16) | (1 << 4) | 1; +pub const DEFAULT_ID_AA64MMFR0_EL1: u64 = (0xf << 24) | 2; pub fn value(register: SystemRegister) -> Option { use SystemRegister::*; Some(match register { - MidrEl1 => 0x410f_d0f0, MpidrEl1 => 0x8000_0000, CtrEl0 => { (1 << 31) | (1 << 29) | (1 << 28) | (4 << 24) | (4 << 20) | (4 << 16) | (3 << 14) | 4 } DczidEl0 => 4, CntfrqEl0 => counter_hz, - IdAa64pfr0El1 => (0xf << 20) | (0xf << 16) | (1 << 4) | 1, IdAa64dfr0El1 => 6, - IdAa64mmfr0El1 => (0xf << 24) | 2, IdAa64isar0El1 => 2 << 20, IdAa64isar1El1 => 1 << 20, ClidrEl1 => CLIDR, @@ -47,11 +49,11 @@ mod tests { use SystemRegister::*; assert_eq!(value(CntfrqEl0), Some(62_500_000)); assert_eq!(value(DczidEl0), Some(4)); - let pfr0 = value(IdAa64pfr0El1).unwrap(); + let pfr0 = DEFAULT_ID_AA64PFR0_EL1; assert_eq!(pfr0 & 0xffff, 0x11); assert_eq!((pfr0 >> 16) & 0xff, 0xff); assert_eq!((pfr0 >> 24) & 15, 0); - let mmfr0 = value(IdAa64mmfr0El1).unwrap(); + let mmfr0 = DEFAULT_ID_AA64MMFR0_EL1; assert_eq!(mmfr0 & 15, 2); assert_eq!((mmfr0 >> 24) & 15, 15); assert_eq!((mmfr0 >> 20) & 15, 0); diff --git a/src/emulator/src/aarch64/translate.rs b/src/emulator/src/aarch64/translate.rs index 9c4cedc..abba353 100644 --- a/src/emulator/src/aarch64/translate.rs +++ b/src/emulator/src/aarch64/translate.rs @@ -26,7 +26,6 @@ pub struct Entry { pub generation: u64, pub ap: u8, pub executable: bool, - pub faulted: bool, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] struct Context { @@ -52,8 +51,6 @@ pub struct Tlb { pub entries: [Entry; 256], pub generation: u64, context: Option, - /// Some entry caches a translation fault. - has_faulted: bool, } impl Default for Tlb { fn default() -> Self { @@ -61,7 +58,6 @@ impl Default for Tlb { entries: [Entry::default(); 256], generation: 1, context: None, - has_faulted: false, } } } @@ -71,27 +67,10 @@ impl Tlb { } pub fn flush(&mut self) { self.generation = self.generation.wrapping_add(1); - self.has_faulted = false; if self.generation <= 1 { self.entries = [Entry::default(); 256]; } } - /// Drop cached translation faults, keeping every valid translation (and so - /// the generation that code and data fast paths are validated against). - /// A context-synchronization event (ISB) makes earlier page-table writes - /// visible to the walker, which can turn a fault into a mapping. It does - /// not invalidate valid translations; only TLBI does. - pub fn purge_faults(&mut self) { - if !self.has_faulted { - return; - } - self.has_faulted = false; - for entry in &mut self.entries { - if entry.faulted { - *entry = Entry::default(); - } - } - } fn slot(virtual_address: u64) -> usize { ((virtual_address >> 12) & 255) as usize } @@ -166,9 +145,6 @@ pub fn translate( let slot = Tlb::slot(virtual_address); let entry = &tlb.entries[slot]; if entry.generation == tlb.generation && entry.virtual_address == page { - if entry.faulted { - return Err(TranslateError::TranslationFault); - } // A cached entry that denies the access is re-walked below: the // descriptor may have been upgraded since it was cached, which // hardware also resolves by retrying the walk. @@ -176,22 +152,10 @@ pub fn translate( return Ok(entry.physical | (virtual_address & PAGE_MASK)); } } - let found = match walk(cpu, memory, virtual_address, access, upper) { - Ok(found) => found, - Err(fault) => { - // Only access-independent failures may be cached as unmapped. - if fault != TranslateError::PermissionFault { - tlb.entries[slot] = Entry { - virtual_address: page, - generation: tlb.generation, - faulted: true, - ..Entry::default() - }; - tlb.has_faulted = true; - } - return Err(fault); - } - }; + // Faults are not cached. ARMv8 never caches an invalid descriptor, so a mapping + // installed later, without a TLBI or an ISB (as Linux's `ioremap` does), must be + // seen by the next access. + let found = walk(cpu, memory, virtual_address, access, upper)?; tlb.entries[slot] = Entry { virtual_address: page, physical: found.physical & !PAGE_MASK, @@ -509,7 +473,8 @@ mod tests { translate(&mut tlb, &cpu, &mut memory, 0, Access::Read), Err(TranslateError::TranslationFault) ); - assert_eq!(memory.reads.get(), count); + // A fault is not cached: the repeated access walks the tables again. + assert!(memory.reads.get() > count); tlb.flush(); memory.put(0, 0x1013); assert_eq!( diff --git a/src/emulator/src/devices/aic.rs b/src/emulator/src/devices/aic.rs new file mode 100644 index 0000000..2a3e26e --- /dev/null +++ b/src/emulator/src/devices/aic.rs @@ -0,0 +1,470 @@ +//! Apple Interrupt Controller, AIC v1: the interrupt controller of the t8103 +//! (M1) SoC, as the Asahi Linux driver drives it on the `apple,t8103-aic` path. +//! +//! The guest sees 896 level-triggered hardware IRQ lines. Each IRQ has a mask +//! bit, a software trigger ORed with its line, and a target CPU. Each CPU has +//! one event register. Reading it returns the lowest-numbered IRQ that is +//! pending, unmasked and targeted at that CPU, and the read itself masks that +//! IRQ (auto-ack). The driver unmasks it again in its EOI handler. Because the +//! IRQ stays masked until then, a level line that is still high does not +//! re-enter the handler. +//! +//! IPIs on this SoC go through the fast-IPI system registers, which are not +//! part of this device. The driver still writes the MMIO IPI registers during +//! CPU bring-up (ACK, and MASK_SET for the SELF and OTHER bits). Those writes +//! are accepted: the mask registers store state and read it back. SEND, ACK, +//! and the per-CPU SET/CLR triggers are write-only no-ops, since no MMIO IPI is +//! ever sent on this path. +//! +//! Reset state: every IRQ is masked and targeted at CPU 0. The driver unmasks +//! an IRQ when it is requested. A guest never sees an IRQ that has not been +//! explicitly enabled. + +/// Mapping length of the AIC register window (the DT `reg` length). +pub const LEN: u64 = 0x8000; + +/// Hardware IRQ lines the guest sees. `AIC_INFO` reports this value, and the +/// driver's header comment documents 896 level-triggered hardware IRQs. Any IRQ +/// number at or above this is not a real pin: it has no effect on delivery, and +/// its register slots read as zero. +const NR_IRQ: usize = 896; +/// 32-bit words that cover `NR_IRQ` in each per-IRQ bitmap. +const IRQ_WORDS: usize = NR_IRQ / 32; + +const INFO: u64 = 0x0004; +const WHOAMI: u64 = 0x2000; +const EVENT: u64 = 0x2004; +const IPI_SEND: u64 = 0x2008; +const IPI_ACK: u64 = 0x200c; +const IPI_MASK_SET: u64 = 0x2024; +const IPI_MASK_CLR: u64 = 0x2028; + +/// `TARGET_CPU`: one word per IRQ at `0x3000 + 4 * irq`. The word holds the CPU +/// bit(s) that receive the IRQ. The driver writes `BIT(cpu)`. +const TARGET_CPU: u64 = 0x3000; + +/// Per-IRQ bitmap register blocks. The driver lays them out in this order, and +/// each block is 32 words (128 bytes) long, one bit per IRQ. +const SW_SET: u64 = 0x4000; +#[cfg(test)] +const SW_CLR: u64 = 0x4080; +#[cfg(test)] +const MASK_SET: u64 = 0x4100; +#[cfg(test)] +const MASK_CLR: u64 = 0x4180; +const HW_STATE: u64 = 0x4200; +const BITMAP_STRIDE: u64 = 0x80; + +/// Explicit per-CPU IPI views: `0x5000 + cpu * 0x80`, with the registers at +/// fixed offsets within each 128-byte block. +const CPU_IPI_BASE: u64 = 0x5000; +const CPU_IPI_STRIDE: u64 = 0x80; +const CPU_IPI_SET: u64 = 0x08; +const CPU_IPI_CLR: u64 = 0x0c; +const CPU_IPI_MASK_SET: u64 = 0x24; +const CPU_IPI_MASK_CLR: u64 = 0x28; + +/// Event word for an IRQ: type `IRQ` (1) in bits 23:16, IRQ number in 15:0. +const EVENT_TYPE_IRQ: u32 = 1 << 16; + +/// Which per-IRQ bitmap a register offset selects. +#[derive(Clone, Copy)] +enum Bitmap { + SwSet, + SwClr, + MaskSet, + MaskClr, + HwState, +} + +/// MMIO IPI registers, both the default view and the explicit per-CPU view. +#[derive(Clone, Copy)] +enum IpiReg { + Send, + Ack, + Set, + Clr, + MaskSet, + MaskClr, +} + +/// A decoded, word-aligned register offset. +#[derive(Clone, Copy)] +enum Reg { + Info, + WhoAmI, + Event, + Target(usize), + Bitmap(Bitmap, usize), + Ipi { cpu: u32, reg: IpiReg }, + Reserved, +} + +/// Apple Interrupt Controller (AIC v1) state for `cpus` CPUs. +pub struct Aic { + cpus: u32, + /// Hardware line state, one bit per IRQ. + hw: [u32; IRQ_WORDS], + /// Software trigger state (`SW_SET`/`SW_CLR`), one bit per IRQ. + sw: [u32; IRQ_WORDS], + /// Mask state (`MASK_SET`/`MASK_CLR`), one bit per IRQ. A set bit masks. + mask: [u32; IRQ_WORDS], + /// CPU routing per IRQ (`TARGET_CPU`). Only the first `NR_IRQ` entries are + /// meaningful. + target: [u32; NR_IRQ], + /// Per-CPU IPI mask state, indexed by CPU. Only the first `cpus` entries + /// are reachable. + ipi_mask: [u32; 32], +} + +/// Mask of the low `n` bytes (`n` in 1..=4) of a 32-bit word. +fn byte_mask(n: u32) -> u32 { + if n >= 4 { + u32::MAX + } else { + (1u32 << (n * 8)) - 1 + } +} + +impl Aic { + /// `cpus` = number of CPUs wired to the IRQ pins (affects per-CPU IPI registers and target masks). + /// + /// Panics if `cpus` is outside `1..=32`, since target registers are 32-bit CPU masks. + pub fn new(cpus: u32) -> Self { + assert!((1..=32).contains(&cpus), "AIC routes to 1..=32 CPUs"); + Self { + cpus, + hw: [0; IRQ_WORDS], + sw: [0; IRQ_WORDS], + mask: [u32::MAX; IRQ_WORDS], + target: [1; NR_IRQ], + ipi_mask: [u32::MAX; 32], + } + } + + /// Drive hardware IRQ `irq` (level-triggered source line). + /// + /// IRQ numbers at or above `NR_IRQ` have no pin and are ignored. + pub fn set_irq(&mut self, irq: u32, asserted: bool) { + let irq = irq as usize; + if irq >= NR_IRQ { + return; + } + let (w, bit) = (irq / 32, 1u32 << (irq % 32)); + if asserted { + self.hw[w] |= bit; + } else { + self.hw[w] &= !bit; + } + } + + /// Whether the IRQ pin of CPU `cpu` is asserted right now (an enabled, unmasked, targeted source is pending). + pub fn irq_pending(&self, cpu: u32) -> bool { + self.first_pending(cpu).is_some() + } + + /// MMIO read at `offset` from the AIC base, performed by CPU `cpu` (AIC_WHOAMI and AIC_EVENT are per reading CPU). Takes &mut self because reading AIC_EVENT has side effects. + /// + /// `size` is 1, 2, 4 or 8 bytes. Accesses that cross a 32-bit register are + /// split per register. Unmapped offsets read as zero. + pub fn read(&mut self, cpu: u32, offset: u64, size: u8) -> u64 { + let size = u64::from(size.min(8)); + let mut value = 0u64; + let mut done = 0u64; + while done < size { + let addr = offset.wrapping_add(done); + let lane = (addr & 3) as u32; + let n = (4 - u64::from(lane)).min(size - done) as u32; + let word = self.read_word(cpu, addr & !3); + let bytes = u64::from((word >> (lane * 8)) & byte_mask(n)); + value |= bytes << (done * 8); + done += u64::from(n); + } + value + } + + /// MMIO write. Only the low `size` bytes of `value` are written. Writes to + /// set/clear registers only affect the bits they name. Writes to read-only + /// or unmapped offsets are ignored. + pub fn write(&mut self, cpu: u32, offset: u64, size: u8, value: u64) { + let size = u64::from(size.min(8)); + let mut done = 0u64; + while done < size { + let addr = offset.wrapping_add(done); + let lane = (addr & 3) as u32; + let n = (4 - u64::from(lane)).min(size - done) as u32; + let lanes = byte_mask(n) << (lane * 8); + let bits = (((value >> (done * 8)) as u32) << (lane * 8)) & lanes; + self.write_word(cpu, addr & !3, bits, lanes); + done += u64::from(n); + } + } + + /// Lowest IRQ that is pending, unmasked, and targeted at `cpu`. A CPU index + /// of 32 or more has no routing bit, so it never matches. + fn first_pending(&self, cpu: u32) -> Option { + let bit = 1u32.checked_shl(cpu).unwrap_or(0); + for w in 0..IRQ_WORDS { + let mut candidates = (self.hw[w] | self.sw[w]) & !self.mask[w]; + while candidates != 0 { + let irq = w * 32 + candidates.trailing_zeros() as usize; + if self.target[irq] & bit != 0 { + return Some(irq); + } + candidates &= candidates - 1; + } + } + None + } + + /// Reading AIC_EVENT: deliver the lowest pending IRQ for `cpu` and mask it + /// (auto-ack). Returns 0 when nothing is pending. + fn claim(&mut self, cpu: u32) -> u32 { + let Some(irq) = self.first_pending(cpu) else { + return 0; + }; + self.mask[irq / 32] |= 1 << (irq % 32); + EVENT_TYPE_IRQ | irq as u32 + } + + fn own_ipi(&self, cpu: u32, reg: IpiReg) -> Reg { + if cpu < self.cpus { + Reg::Ipi { cpu, reg } + } else { + Reg::Reserved + } + } + + fn decode(&self, offset: u64, cpu: u32) -> Reg { + match offset { + INFO => Reg::Info, + WHOAMI => Reg::WhoAmI, + EVENT => Reg::Event, + IPI_SEND => self.own_ipi(cpu, IpiReg::Send), + IPI_ACK => self.own_ipi(cpu, IpiReg::Ack), + IPI_MASK_SET => self.own_ipi(cpu, IpiReg::MaskSet), + IPI_MASK_CLR => self.own_ipi(cpu, IpiReg::MaskClr), + o if (TARGET_CPU..SW_SET).contains(&o) => { + let irq = ((o - TARGET_CPU) / 4) as usize; + if irq < NR_IRQ { + Reg::Target(irq) + } else { + Reg::Reserved + } + } + o if (SW_SET..HW_STATE + BITMAP_STRIDE).contains(&o) => { + let block = (o - SW_SET) / BITMAP_STRIDE; + let word = (((o - SW_SET) % BITMAP_STRIDE) / 4) as usize; + let which = match block { + 0 => Bitmap::SwSet, + 1 => Bitmap::SwClr, + 2 => Bitmap::MaskSet, + 3 => Bitmap::MaskClr, + _ => Bitmap::HwState, + }; + if word < IRQ_WORDS { + Reg::Bitmap(which, word) + } else { + Reg::Reserved + } + } + o if (CPU_IPI_BASE..CPU_IPI_BASE + CPU_IPI_STRIDE * u64::from(self.cpus)) + .contains(&o) => + { + let rel = o - CPU_IPI_BASE; + let reg = match rel % CPU_IPI_STRIDE { + CPU_IPI_SET => IpiReg::Set, + CPU_IPI_CLR => IpiReg::Clr, + CPU_IPI_MASK_SET => IpiReg::MaskSet, + CPU_IPI_MASK_CLR => IpiReg::MaskClr, + _ => return Reg::Reserved, + }; + Reg::Ipi { + cpu: (rel / CPU_IPI_STRIDE) as u32, + reg, + } + } + _ => Reg::Reserved, + } + } + + /// Read the 32-bit register at word-aligned `offset`. + fn read_word(&mut self, cpu: u32, offset: u64) -> u32 { + match self.decode(offset, cpu) { + Reg::Info => NR_IRQ as u32, + Reg::WhoAmI => cpu, + Reg::Event => self.claim(cpu), + Reg::Target(irq) => self.target[irq], + Reg::Bitmap(Bitmap::SwSet | Bitmap::SwClr, w) => self.sw[w], + Reg::Bitmap(Bitmap::MaskSet | Bitmap::MaskClr, w) => self.mask[w], + Reg::Bitmap(Bitmap::HwState, w) => self.hw[w], + Reg::Ipi { + cpu: c, + reg: IpiReg::MaskSet | IpiReg::MaskClr, + } => self.ipi_mask[c as usize], + Reg::Ipi { .. } | Reg::Reserved => 0, + } + } + + /// Write the lanes in `lanes` of the 32-bit register at word-aligned + /// `offset`. `bits` holds the written value already shifted into those + /// lanes and zero elsewhere. + fn write_word(&mut self, cpu: u32, offset: u64, bits: u32, lanes: u32) { + match self.decode(offset, cpu) { + Reg::Target(irq) => self.target[irq] = (self.target[irq] & !lanes) | bits, + Reg::Bitmap(Bitmap::SwSet, w) => self.sw[w] |= bits, + Reg::Bitmap(Bitmap::SwClr, w) => self.sw[w] &= !bits, + Reg::Bitmap(Bitmap::MaskSet, w) => self.mask[w] |= bits, + Reg::Bitmap(Bitmap::MaskClr, w) => self.mask[w] &= !bits, + Reg::Ipi { + cpu: c, + reg: IpiReg::MaskSet, + } => self.ipi_mask[c as usize] |= bits, + Reg::Ipi { + cpu: c, + reg: IpiReg::MaskClr, + } => self.ipi_mask[c as usize] &= !bits, + // Read-only registers, write-only IPI triggers, and reserved + // offsets: accepted and ignored. + Reg::Info + | Reg::WhoAmI + | Reg::Event + | Reg::Bitmap(Bitmap::HwState, _) + | Reg::Ipi { .. } + | Reg::Reserved => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Address of the MASK_CLR bit for `irq`. + fn mask_clr(irq: u32) -> u64 { + MASK_CLR + 4 * u64::from(irq / 32) + } + + /// Clear the mask of `irq` on CPU 0, as the driver does when it enables an IRQ. + fn unmask(aic: &mut Aic, irq: u32) { + aic.write(0, mask_clr(irq), 4, 1u64 << (irq % 32)); + } + + fn event(aic: &mut Aic, cpu: u32) -> u64 { + aic.read(cpu, EVENT, 4) + } + + #[test] + fn event_returns_lowest_pending_irq_and_auto_masks() { + let mut aic = Aic::new(1); + unmask(&mut aic, 40); + unmask(&mut aic, 5); + aic.set_irq(40, true); + aic.set_irq(5, true); + + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 5)); + // IRQ 5 was auto-masked by the read, so IRQ 40 is next. + assert!(aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 40)); + assert!(!aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), 0); + } + + #[test] + fn masked_irq_does_not_assert_pin_until_unmasked() { + let mut aic = Aic::new(1); + aic.set_irq(3, true); + assert!(!aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), 0); + + unmask(&mut aic, 3); + assert!(aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 3)); + } + + #[test] + fn reasserting_while_auto_masked_does_not_raise_pin() { + let mut aic = Aic::new(1); + unmask(&mut aic, 3); + aic.set_irq(3, true); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 3)); + + // The line drops and rises again while the IRQ is auto-masked. + aic.set_irq(3, false); + aic.set_irq(3, true); + assert!(!aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), 0); + } + + #[test] + fn mask_clr_reenables_and_pin_rises_again() { + let mut aic = Aic::new(1); + unmask(&mut aic, 3); + aic.set_irq(3, true); + event(&mut aic, 0); + assert!(!aic.irq_pending(0)); + + // The driver's EOI unmasks the level-triggered source, which is still high. + unmask(&mut aic, 3); + assert!(aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 3)); + } + + #[test] + fn target_register_routes_irq_away_from_reading_cpu() { + let mut aic = Aic::new(2); + unmask(&mut aic, 7); + aic.set_irq(7, true); + // Reset routes every IRQ to CPU 0. + assert!(aic.irq_pending(0)); + + // The driver writes BIT(1) to route IRQ 7 to CPU 1. + aic.write(0, TARGET_CPU + 4 * 7, 4, 1 << 1); + assert!(!aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), 0); + assert!(aic.irq_pending(1)); + assert_eq!(event(&mut aic, 1), u64::from(EVENT_TYPE_IRQ | 7)); + } + + #[test] + fn software_trigger_asserts_without_hardware_line() { + let mut aic = Aic::new(1); + unmask(&mut aic, 40); + // IRQ 40 is bit 8 of the second word of each bitmap block. + aic.write(0, SW_SET + 4, 4, 1 << 8); + assert!(aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 40)); + + aic.write(0, SW_SET + 4, 4, 1 << 8); + aic.write(0, SW_CLR + 4, 4, 1 << 8); + assert!(!aic.irq_pending(0)); + } + + #[test] + fn whoami_reports_the_reading_cpu() { + let mut aic = Aic::new(2); + assert_eq!(aic.read(0, WHOAMI, 4), 0); + assert_eq!(aic.read(1, WHOAMI, 4), 1); + } + + #[test] + fn unknown_offsets_read_zero_and_ignore_writes() { + let mut aic = Aic::new(1); + // Reserved gap, AIC_CONFIG (not used by the driver on this path), a + // reserved word in the mask block, a target slot past the last IRQ, a + // bitmap block past HW_STATE, and a per-CPU view for a CPU that is absent. + let offsets = [ + 0x0000, + 0x0010, + 0x2010, + MASK_SET + 4 * 28, + TARGET_CPU + 4 * NR_IRQ as u64, + HW_STATE + BITMAP_STRIDE, + CPU_IPI_BASE + CPU_IPI_STRIDE, + ]; + for offset in offsets { + aic.write(0, offset, 4, u64::from(u32::MAX)); + assert_eq!(aic.read(0, offset, 4), 0, "offset {offset:#x}"); + } + } +} diff --git a/src/emulator/src/devices/mod.rs b/src/emulator/src/devices/mod.rs index 72ce939..fa62444 100644 --- a/src/emulator/src/devices/mod.rs +++ b/src/emulator/src/devices/mod.rs @@ -1,7 +1,10 @@ +pub mod aic; pub mod bus; pub mod gicv2; pub mod gicv3; pub mod pl011; +pub mod pmgr; +pub mod s5l_uart; pub mod virtio_blk; pub use bus::{Bus, BusDevice, Device}; diff --git a/src/emulator/src/devices/pmgr.rs b/src/emulator/src/devices/pmgr.rs new file mode 100644 index 0000000..a2cba60 --- /dev/null +++ b/src/emulator/src/devices/pmgr.rs @@ -0,0 +1,93 @@ +//! Apple's power manager (PMGR), as the M1 Linux power-domain driver drives it. +//! +//! Each power domain has one 32-bit power-state register (`apple-pmgr-pwrstate`): +//! `PS_TARGET` in bits 3:0 is the state the driver asks for, and `PS_ACTUAL` in bits +//! 7:4 is the state the hardware is in. The driver writes the target and polls until +//! the actual state matches. This model acknowledges a write at once, so the actual +//! state is the target, and every register starts in the active state iBoot leaves +//! the always-on domains in. Other bits are stored as written. +use alloc::vec; +use alloc::vec::Vec; + +/// `PS_ACTIVE`: the target and the actual state both name the active state. +const ACTIVE: u32 = 0xff; +const TARGET: u32 = 0x0f; +const ACTUAL: u32 = 0xf0; + +/// The power-state registers of one PMGR block of `len` bytes. +pub struct Pmgr { + words: Vec, +} + +impl Pmgr { + /// A block of `len` bytes (a multiple of four) with every power domain active. + pub fn new(len: u64) -> Self { + Self { + words: vec![ACTIVE; (len / 4) as usize], + } + } + + /// Read `size` bytes at `offset` (relative to the block base). + pub fn read(&self, offset: u64, size: u8) -> u64 { + (0..u64::from(size)).fold(0, |value, i| { + let at = offset + i; + let word = self.words[(at / 4) as usize]; + value | u64::from((word >> (8 * (at % 4))) as u8) << (8 * i) + }) + } + + /// Write the low `size` bytes of `value` at `offset`. Each 32-bit register the write + /// touches takes its target power state at once. + pub fn write(&mut self, offset: u64, size: u8, value: u64) { + for i in 0..u64::from(size) { + let at = offset + i; + let shift = 8 * (at % 4); + let byte = ((value >> (8 * i)) & 0xff) as u32; + let word = &mut self.words[(at / 4) as usize]; + *word = (*word & !(0xff << shift)) | byte << shift; + } + let first = (offset / 4) as usize; + let last = ((offset + u64::from(size) - 1) / 4) as usize; + for word in &mut self.words[first..=last] { + *word = (*word & !ACTUAL) | (*word & TARGET) << 4; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PS: u64 = 0x100; + const LEN: u64 = 0x4000; + + #[test] + fn every_power_domain_starts_active() { + let pmgr = Pmgr::new(LEN); + assert_eq!(pmgr.read(PS, 4) & 0xff, 0xff); + } + + #[test] + fn a_target_is_acknowledged_as_the_actual_state_at_once() { + let mut pmgr = Pmgr::new(LEN); + pmgr.write(PS, 4, 0x0); + let state = pmgr.read(PS, 4); + assert_eq!(state & 0x0f, 0, "target is what was written"); + assert_eq!((state >> 4) & 0x0f, 0, "actual follows the target"); + pmgr.write(PS, 4, 0x0f | 1 << 31); + assert_eq!( + pmgr.read(PS, 4) & 0xff, + 0xff, + "active again, reset bit stored" + ); + } + + #[test] + fn narrow_writes_change_only_their_bytes_and_flags_are_stored() { + let mut pmgr = Pmgr::new(LEN); + // WAS_PWRGATED (bit 8) lives in the second byte: a byte write leaves the rest. + pmgr.write(PS + 1, 1, 0x1); + assert_eq!(pmgr.read(PS + 1, 1), 0x1); + assert_eq!(pmgr.read(PS, 1), 0xff); + } +} diff --git a/src/emulator/src/devices/s5l_uart.rs b/src/emulator/src/devices/s5l_uart.rs new file mode 100644 index 0000000..6566e3b --- /dev/null +++ b/src/emulator/src/devices/s5l_uart.rs @@ -0,0 +1,358 @@ +//! Apple S5L UART (`apple,s5l-uart`), the console port of M1-class SoCs. +//! +//! The Apple variant differs from the classic Samsung register map in one way +//! that matters here: it has no `UINTP`/`UINTM` interrupt registers. Its +//! interrupt causes are the flag bits of `UTRSTAT` itself. A cause is cleared +//! by writing a one to its bit, and a `UCON` enable bit decides whether a +//! pending cause reaches the line. `apple_serial_handle_irq` in the driver +//! works this way and never touches `UINTP`. +//! +//! Transmit completes at once. Every byte written to `UTXH` reaches the sink, +//! and the transmit FIFO never reports full. The one transmit cause is +//! `TXTHRESH` (`UTRSTAT` bit 5). Real hardware latches it when the transmit +//! FIFO drops below its threshold, and in this model every transmit does that, +//! so the cause is raised after each byte. The line rises only while +//! `UCON.TXTHRESH_ENA` is set, which is when the driver wants a TX interrupt. +//! +//! The driver's TX path never waits on that interrupt. `enable_tx_pio` sets the +//! enable bit and calls the TX routine itself, because the Apple UART only +//! raises edge-triggered TX interrupts (see the comment there). That routine +//! drains the queue synchronously, since the FIFO never fills, and then clears +//! the enable bit through `stop_tx`. If an interrupt does arrive while work is +//! left, the handler clears `TXTHRESH` first and then writes more bytes, which +//! raises the cause again, so each entry drains some of the queue. +//! +//! Receive has no input in this model: the receive FIFO is empty, `RXDR` stays +//! clear, and no receive cause is ever raised. + +use alloc::boxed::Box; +use alloc::collections::VecDeque; + +/// Mapping length of the UART register window. +pub const LEN: u64 = 0x1000; + +const ULCON: u64 = 0x00; +const UCON: u64 = 0x04; +const UFCON: u64 = 0x08; +const UMCON: u64 = 0x0c; +const UTRSTAT: u64 = 0x10; +const UFSTAT: u64 = 0x18; +const UTXH: u64 = 0x20; +const URXH: u64 = 0x24; +const UBRDIV: u64 = 0x28; +const UFRACVAL: u64 = 0x2c; + +/// `UCON` bit that enables the transmit interrupt line, `APPLE_S5L_UCON_TXTHRESH_ENA`. +const UCON_TXTHRESH_ENA: u32 = 1 << 13; + +/// `UTRSTAT` bit for the transmit-threshold cause, `APPLE_S5L_UTRSTAT_TXTHRESH`. +const UTRSTAT_TXTHRESH: u32 = 1 << 5; +/// Transmitter empty and transmit FIFO empty. Always set: transmit is instant. +const UTRSTAT_TXE: u32 = 1 << 2; +const UTRSTAT_TXFE: u32 = 1 << 1; +/// Receive data ready, set only while the receive FIFO holds data. +const UTRSTAT_RXDR: u32 = 1 << 0; + +/// `UFCON` bits that reset a FIFO. The hardware clears them itself, so they +/// never read back as set. +const UFCON_RESETRX: u32 = 1 << 1; +const UFCON_RESETTX: u32 = 1 << 2; + +/// `UFSTAT` receive count field and its full flag. +const UFSTAT_RXMASK: u32 = 0xf; +const UFSTAT_RXFULL: u32 = 1 << 8; +/// Receive FIFO depth reported in `UFSTAT`, `fifosize` for this port. +const FIFO_DEPTH: usize = 16; + +/// An Apple S5L UART. +/// +/// The sink takes each transmitted byte in order. The interrupt line is read +/// through [`S5lUart::irq_asserted`]. +pub struct S5lUart { + sink: Box, + ulcon: u32, + ucon: u32, + ufcon: u32, + umcon: u32, + ubrdiv: u32, + ufracval: u32, + /// Pending interrupt causes, in `UTRSTAT` bit positions. + pending: u32, + rx: VecDeque, +} + +impl S5lUart { + /// A UART writing transmitted bytes to `sink`. + pub fn new(sink: impl FnMut(u8) + Send + 'static) -> Self { + Self { + sink: Box::new(sink), + ulcon: 0, + ucon: 0, + ufcon: 0, + umcon: 0, + ubrdiv: 0, + ufracval: 0, + pending: 0, + rx: VecDeque::new(), + } + } + + /// Read a register. `offset` is relative to the device base; `size` is in + /// bytes. Unknown offsets read as zero. + pub fn read(&mut self, offset: u64, size: u8) -> u64 { + let value = match offset { + ULCON => self.ulcon, + UCON => self.ucon, + UFCON => self.ufcon, + UMCON => self.umcon, + UTRSTAT => self.utrstat(), + UFSTAT => self.ufstat(), + URXH => self.rx.pop_front().map_or(0, u32::from), + UBRDIV => self.ubrdiv, + UFRACVAL => self.ufracval, + _ => 0, + }; + u64::from(value) & lane_mask(size) + } + + /// Write a register. Only the low `size` bytes of `value` are used. + /// Unknown offsets and read-only registers ignore the write. + pub fn write(&mut self, offset: u64, size: u8, value: u64) { + let value = (value & lane_mask(size)) as u32; + match offset { + ULCON => self.ulcon = value, + UCON => self.ucon = value, + UFCON => { + if value & UFCON_RESETRX != 0 { + self.rx.clear(); + } + self.ufcon = value & !(UFCON_RESETRX | UFCON_RESETTX); + } + UMCON => self.umcon = value, + // Write-one-to-clear. Bits that are not causes have nothing pending + // to clear, so they fall through harmlessly. + UTRSTAT => self.pending &= !value, + UTXH => self.transmit(value as u8), + UBRDIV => self.ubrdiv = value, + UFRACVAL => self.ufracval = value, + _ => {} + } + } + + /// Level of the interrupt line to the AIC: asserted while the transmit + /// cause is pending and its `UCON` enable bit is set. + pub fn irq_asserted(&self) -> bool { + self.pending & UTRSTAT_TXTHRESH != 0 && self.ucon & UCON_TXTHRESH_ENA != 0 + } + + fn transmit(&mut self, byte: u8) { + (self.sink)(byte); + self.pending |= UTRSTAT_TXTHRESH; + } + + fn utrstat(&self) -> u32 { + let rxdr = if self.rx.is_empty() { 0 } else { UTRSTAT_RXDR }; + self.pending | UTRSTAT_TXE | UTRSTAT_TXFE | rxdr + } + + /// The transmit FIFO never holds anything, so only the receive count shows. + fn ufstat(&self) -> u32 { + let len = self.rx.len(); + let full = if len >= FIFO_DEPTH { UFSTAT_RXFULL } else { 0 }; + full | (len as u32 & UFSTAT_RXMASK) + } + + /// Feed one byte into the receive FIFO, as input from a host would. + #[cfg(test)] + fn feed_rx(&mut self, byte: u8) { + self.rx.push_back(byte); + } +} + +/// Mask for the low `size` bytes of a register access. +fn lane_mask(size: u8) -> u64 { + match size { + 1 => 0xff, + 2 => 0xffff, + 4 => 0xffff_ffff, + _ => u64::MAX, + } +} + +#[cfg(test)] +mod tests { + use super::{ + S5lUart, UBRDIV, UCON, UCON_TXTHRESH_ENA, UFCON, UFRACVAL, UFSTAT, ULCON, UMCON, URXH, + UTRSTAT, UTRSTAT_TXTHRESH, UTXH, + }; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; + + const CAPACITY: usize = 512; + const ZERO: AtomicU8 = AtomicU8::new(0); + + /// Bytes the sink received, shared with the test through atomics so the + /// sink stays `Send` without `std`. + struct Capture { + len: AtomicUsize, + bytes: [AtomicU8; CAPACITY], + } + + impl Capture { + fn new() -> Arc { + Arc::new(Self { + len: AtomicUsize::new(0), + bytes: [ZERO; CAPACITY], + }) + } + + fn push(&self, byte: u8) { + let index = self.len.fetch_add(1, Ordering::SeqCst); + self.bytes[index].store(byte, Ordering::SeqCst); + } + + fn collected(&self) -> alloc::vec::Vec { + (0..self.len.load(Ordering::SeqCst)) + .map(|i| self.bytes[i].load(Ordering::SeqCst)) + .collect() + } + } + + fn uart_with_capture() -> (S5lUart, Arc) { + let capture = Capture::new(); + let sink = Arc::clone(&capture); + (S5lUart::new(move |byte| sink.push(byte)), capture) + } + + /// The `UFCON` value the driver's startup writes: FIFO mode, RX trigger 8, + /// and the RX reset bit it ORs in (plus TX reset when not a console). + const UFCON_STARTUP: u32 = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 4); + + #[test] + fn transmitted_bytes_reach_sink_in_order() { + let (mut uart, capture) = uart_with_capture(); + for &byte in b"volt\n" { + uart.write(UTXH, 4, u64::from(byte)); + } + // Only the low byte of the register is the character. + uart.write(UTXH, 4, 0x1_0000_0041); + uart.write(UTXH, 1, 0x142); + assert_eq!(capture.collected(), b"volt\nAB"); + } + + #[test] + fn transmit_fifo_never_reports_full_or_nonempty() { + let (mut uart, _capture) = uart_with_capture(); + for i in 0..200u64 { + uart.write(UTXH, 4, i); + let ufstat = uart.read(UFSTAT, 4); + assert_eq!(ufstat & (1 << 9), 0, "TXFULL set after byte {i}"); + assert_eq!(ufstat & (0xf << 4), 0, "TX count nonzero after byte {i}"); + } + } + + #[test] + fn early_console_sees_room_and_empty_transmitter() { + // `samsung_early_putc` polls TXFULL when UFCON has FIFO mode set, and + // TXFE in UTRSTAT otherwise. The driver's startup sets FIFO mode. + let (mut uart, _capture) = uart_with_capture(); + uart.write(UFCON, 4, u64::from(UFCON_STARTUP)); + uart.write(UTXH, 4, u64::from(b'x')); + assert_eq!(uart.read(UFSTAT, 4) & (1 << 9), 0); + assert_ne!(uart.read(UTRSTAT, 4) & (1 << 1), 0); + // Without FIFO mode the early console polls TXFE only; it must also pass. + uart.write(UFCON, 4, 0); + assert_ne!(uart.read(UTRSTAT, 4) & (1 << 1), 0); + } + + #[test] + fn write_one_to_clear_drops_the_line() { + let (mut uart, _capture) = uart_with_capture(); + uart.write(UCON, 4, u64::from(UCON_TXTHRESH_ENA)); + uart.write(UTXH, 4, u64::from(b'a')); + assert!(uart.irq_asserted()); + assert_ne!(uart.read(UTRSTAT, 4) & u64::from(UTRSTAT_TXTHRESH), 0); + + uart.write(UTRSTAT, 4, u64::from(UTRSTAT_TXTHRESH)); + assert!(!uart.irq_asserted()); + assert_eq!(uart.read(UTRSTAT, 4) & u64::from(UTRSTAT_TXTHRESH), 0); + } + + #[test] + fn enable_bit_masks_the_line_but_not_the_cause() { + let (mut uart, _capture) = uart_with_capture(); + uart.write(UTXH, 4, u64::from(b'a')); + assert!(!uart.irq_asserted(), "no enable bit, no line"); + assert_ne!(uart.read(UTRSTAT, 4) & u64::from(UTRSTAT_TXTHRESH), 0); + + uart.write(UCON, 4, u64::from(UCON_TXTHRESH_ENA)); + assert!(uart.irq_asserted(), "enabling reveals the pending cause"); + } + + #[test] + fn driver_tx_kick_completes_without_waiting_for_an_interrupt() { + // `enable_tx_pio`: set the enable bit, write the queued bytes inline, + // then `stop_tx` clears the enable bit. The line must end released. + let (mut uart, capture) = uart_with_capture(); + uart.write(UCON, 4, u64::from(UCON_TXTHRESH_ENA)); + for &byte in b"boot" { + uart.write(UTXH, 4, u64::from(byte)); + } + uart.write(UCON, 4, 0); + assert!(!uart.irq_asserted()); + assert_eq!(capture.collected(), b"boot"); + + // A later kick with a cause left over from before still makes progress: + // the handler clears the cause, then drains and disables the line. + uart.write(UCON, 4, u64::from(UCON_TXTHRESH_ENA)); + assert!(uart.irq_asserted()); + uart.write(UTRSTAT, 4, u64::from(UTRSTAT_TXTHRESH)); + uart.write(UTXH, 4, u64::from(b'!')); + uart.write(UCON, 4, 0); + assert!(!uart.irq_asserted()); + assert_eq!(capture.collected(), b"boot!"); + } + + #[test] + fn startup_registers_read_back() { + let (mut uart, _capture) = uart_with_capture(); + let ucon = 0x3a85_u32; // APPLE_S5L_UCON_DEFAULT with all four enable bits set + uart.write(ULCON, 4, 0x3); + uart.write(UCON, 4, u64::from(ucon)); + uart.write(UFCON, 4, u64::from(UFCON_STARTUP)); + uart.write(UMCON, 4, 0x10); + uart.write(UBRDIV, 4, 0x1a0); + uart.write(UFRACVAL, 4, 0x7); + + assert_eq!(uart.read(ULCON, 4), 0x3); + assert_eq!(uart.read(UCON, 4), u64::from(ucon)); + // The FIFO reset bits are self-clearing and do not read back. + assert_eq!(uart.read(UFCON, 4), u64::from(UFCON_STARTUP & !0b110)); + assert_eq!(uart.read(UMCON, 4), 0x10); + assert_eq!(uart.read(UBRDIV, 4), 0x1a0); + assert_eq!(uart.read(UFRACVAL, 4), 0x7); + } + + #[test] + fn unknown_offsets_read_zero_and_ignore_writes() { + let (mut uart, _capture) = uart_with_capture(); + uart.write(0x30, 4, 0xffff_ffff); + uart.write(0x38, 4, 0xffff_ffff); + assert_eq!(uart.read(0x30, 4), 0); + assert_eq!(uart.read(0x38, 4), 0); + assert!(!uart.irq_asserted()); + } + + #[test] + fn receive_fifo_drives_data_ready_and_urxh() { + let (mut uart, _capture) = uart_with_capture(); + assert_eq!(uart.read(URXH, 4), 0); + assert_eq!(uart.read(UTRSTAT, 4) & 1, 0); + + uart.feed_rx(b'k'); + assert_ne!(uart.read(UTRSTAT, 4) & 1, 0); + assert_eq!(uart.read(UFSTAT, 4) & 0xf, 1); + assert_eq!(uart.read(URXH, 4), u64::from(b'k')); + assert_eq!(uart.read(UTRSTAT, 4) & 1, 0); + } +} diff --git a/src/emulator/src/system/boot.rs b/src/emulator/src/system/boot.rs index 1dd9a04..bb6d6b6 100644 --- a/src/emulator/src/system/boot.rs +++ b/src/emulator/src/system/boot.rs @@ -43,6 +43,8 @@ pub enum Error { /// A boot option the selected kernel format cannot honour. Unsupported(&'static str), Memory(MemoryError), + /// The device tree the caller supplied cannot be patched. + DeviceTree(super::dtb::Error), } impl core::fmt::Display for Error { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { @@ -163,6 +165,36 @@ pub fn prepare_boot( }) } +/// Place a Linux `Image` at its text offset above `ram_base`, with `tree` (a device +/// tree the caller already built or patched) 2 MiB-aligned after it. The returned +/// layout's `boot_info` is the tree's address, which the kernel receives in `x0`. +pub fn place_with_tree( + memory: &mut impl GuestMemory, + image: &[u8], + tree: &[u8], + ram_base: u64, + ram_size: u64, +) -> Result { + let header = parse(image)?; + let entry = load_address(ram_base, header)?; + let occupied = header.image_size.max(image.len() as u64); + let tree_at = align(entry.checked_add(occupied).ok_or(Error::NoRoom)?)?; + let end = ram_base.checked_add(ram_size).ok_or(Error::NoRoom)?; + if tree_at + .checked_add(tree.len() as u64) + .is_none_or(|tree_end| tree_end > end) + { + return Err(Error::NoRoom); + } + memory.write(entry, image).map_err(Error::Memory)?; + memory.write(tree_at, tree).map_err(Error::Memory)?; + Ok(Layout { + entry, + boot_info: tree_at, + initrd: None, + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/emulator/src/system/dtb.rs b/src/emulator/src/system/dtb.rs new file mode 100644 index 0000000..90804fe --- /dev/null +++ b/src/emulator/src/system/dtb.rs @@ -0,0 +1,794 @@ +//! Flattened device tree patching for a device tree supplied by the user. +//! +//! The kernel reads `/chosen/bootargs` for its command line and `/memory` `reg` +//! for the RAM it may use, so both must describe what the emulator provides. The +//! blob is parsed into an owned tree of nodes and properties, keeping their order +//! and the memory reservation map, the two properties are edited there, and the +//! tree is serialized into a fresh blob. Nothing else in the tree is touched. + +use alloc::vec::Vec; +use core::fmt; + +const MAGIC: u32 = 0xd00d_feed; +const HEADER_LEN: usize = 40; +/// Newest format version this reader and writer speak. +const VERSION: u32 = 17; +const LAST_COMP_VERSION: u32 = 16; +const RESERVE_ENTRY_LEN: usize = 16; + +const FDT_BEGIN_NODE: u32 = 1; +const FDT_END_NODE: u32 = 2; +const FDT_PROP: u32 = 3; +const FDT_NOP: u32 = 4; +const FDT_END: u32 = 9; + +/// Cell counts the spec assumes when the root node omits them. +const DEFAULT_ADDRESS_CELLS: u32 = 2; +const DEFAULT_SIZE_CELLS: u32 = 1; + +const TRUNCATED_STRUCT: Error = Error::Malformed("structure block ends inside a token"); +const PROP_PAST_STRUCT: Error = Error::Malformed("property runs past structure block"); + +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + /// The blob is shorter than its header, or than the `totalsize` it declares. + Truncated { + len: usize, + needed: usize, + }, + BadMagic(u32), + UnsupportedVersion(u32), + /// The blob violates the flattened format; the message names the first problem found. + Malformed(&'static str), + /// No node named `memory*` with `device_type = "memory"` exists to receive RAM. + NoMemoryNode, + /// `#address-cells` or `#size-cells` is not 1 or 2, so `reg` cannot be encoded. + CellCount(u32), + /// The RAM base or size does not fit in the single cell the root node declares. + ValueTooLarge { + value: u64, + cells: u32, + }, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Truncated { len, needed } => { + write!(f, "device tree is {len} bytes, needs {needed}") + } + Error::BadMagic(magic) => write!(f, "bad device tree magic {magic:#010x}"), + Error::UnsupportedVersion(version) => { + write!( + f, + "unsupported device tree version {version}, need 17 or newer" + ) + } + Error::Malformed(what) => write!(f, "malformed device tree: {what}"), + Error::NoMemoryNode => { + write!( + f, + "device tree has no memory node with device_type \"memory\"" + ) + } + Error::CellCount(cells) => { + write!(f, "root cell count {cells} is not 1 or 2") + } + Error::ValueTooLarge { value, cells } => { + write!(f, "{value:#x} does not fit in {cells} cell(s)") + } + } + } +} + +impl core::error::Error for Error {} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Node { + name: Vec, + props: Vec, + children: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Prop { + name: Vec, + value: Vec, +} + +impl Node { + fn named(name: &[u8]) -> Self { + Node { + name: name.to_vec(), + props: Vec::new(), + children: Vec::new(), + } + } +} + +struct Parsed { + reservations: Vec<(u64, u64)>, + boot_cpuid_phys: u32, + root: Node, +} + +/// Return a new blob equal to `blob` except: if `bootargs` is `Some`, `/chosen` +/// gets the `bootargs` string property (replacing any existing one, creating +/// `/chosen` if absent); if `memory` is `Some((base, size))`, the first node +/// (depth-first) whose name starts with `memory` and whose `device_type` is +/// `memory` gets `reg` encoded with the root's `#address-cells` and `#size-cells`. +/// Every other node and property is preserved in order. +pub fn patch( + blob: &[u8], + bootargs: Option<&str>, + memory: Option<(u64, u64)>, +) -> Result, Error> { + let Parsed { + reservations, + boot_cpuid_phys, + mut root, + } = parse(blob)?; + if let Some(args) = bootargs { + let mut value = args.as_bytes().to_vec(); + value.push(0); + set_prop(chosen_node(&mut root), b"bootargs", value); + } + if let Some((base, size)) = memory { + let address_cells = cell_count(&root, b"#address-cells", DEFAULT_ADDRESS_CELLS)?; + let size_cells = cell_count(&root, b"#size-cells", DEFAULT_SIZE_CELLS)?; + let mut reg = Vec::new(); + push_cells(&mut reg, base, address_cells)?; + push_cells(&mut reg, size, size_cells)?; + let node = memory_node_mut(&mut root).ok_or(Error::NoMemoryNode)?; + set_prop(node, b"reg", reg); + } + serialize(&root, &reservations, boot_cpuid_phys) +} + +fn chosen_node(root: &mut Node) -> &mut Node { + let index = match root.children.iter().position(|c| c.name == b"chosen") { + Some(index) => index, + None => { + root.children.push(Node::named(b"chosen")); + root.children.len() - 1 + } + }; + &mut root.children[index] +} + +fn memory_node_mut(node: &mut Node) -> Option<&mut Node> { + if is_memory_device(node) { + return Some(node); + } + node.children.iter_mut().find_map(memory_node_mut) +} + +fn is_memory_device(node: &Node) -> bool { + node.name.starts_with(b"memory") + && prop_value(node, b"device_type") + .is_some_and(|value| value.strip_suffix(b"\0").unwrap_or(value) == b"memory") +} + +fn prop_value<'a>(node: &'a Node, name: &[u8]) -> Option<&'a [u8]> { + node.props + .iter() + .find(|p| p.name == name) + .map(|p| p.value.as_slice()) +} + +fn set_prop(node: &mut Node, name: &[u8], value: Vec) { + match node.props.iter_mut().find(|p| p.name == name) { + Some(prop) => prop.value = value, + None => node.props.push(Prop { + name: name.to_vec(), + value, + }), + } +} + +fn cell_count(root: &Node, name: &[u8], default: u32) -> Result { + match prop_value(root, name) { + None => Ok(default), + Some(value) => <[u8; 4]>::try_from(value) + .map(u32::from_be_bytes) + .map_err(|_| Error::Malformed("cell count property is not one u32")), + } +} + +/// Appends `value` as `cells` big-endian u32s; only 1 and 2 cells are valid. +fn push_cells(out: &mut Vec, value: u64, cells: u32) -> Result<(), Error> { + match cells { + 1 => { + let word = u32::try_from(value).map_err(|_| Error::ValueTooLarge { value, cells })?; + out.extend_from_slice(&word.to_be_bytes()); + Ok(()) + } + 2 => { + out.extend_from_slice(&value.to_be_bytes()); + Ok(()) + } + _ => Err(Error::CellCount(cells)), + } +} + +fn parse(blob: &[u8]) -> Result { + let header = parse_header(blob)?; + let data = blob + .get(..header.totalsize) + .ok_or(Error::Malformed("totalsize exceeds blob"))?; + let reservations = parse_reservations(data, header.off_mem_rsvmap)?; + let structs = block( + data, + header.off_dt_struct, + header.size_dt_struct, + "structure block lies outside totalsize", + )?; + let strings = block( + data, + header.off_dt_strings, + header.size_dt_strings, + "strings block lies outside totalsize", + )?; + if header.off_dt_struct % 4 != 0 { + return Err(Error::Malformed("structure block is not 4-byte aligned")); + } + let root = parse_struct(structs, strings)?; + Ok(Parsed { + reservations, + boot_cpuid_phys: header.boot_cpuid_phys, + root, + }) +} + +struct Header { + totalsize: usize, + off_dt_struct: u32, + off_dt_strings: u32, + off_mem_rsvmap: u32, + size_dt_strings: u32, + size_dt_struct: u32, + boot_cpuid_phys: u32, +} + +fn parse_header(blob: &[u8]) -> Result { + if blob.len() < HEADER_LEN { + return Err(Error::Truncated { + len: blob.len(), + needed: HEADER_LEN, + }); + } + let word = |at: usize| read_u32(blob, at).ok_or(Error::Malformed("header is truncated")); + let magic = word(0)?; + if magic != MAGIC { + return Err(Error::BadMagic(magic)); + } + let totalsize = word(4)? as usize; + let version = word(20)?; + if version < VERSION { + return Err(Error::UnsupportedVersion(version)); + } + if totalsize < HEADER_LEN { + return Err(Error::Malformed("totalsize is smaller than the header")); + } + if totalsize > blob.len() { + return Err(Error::Truncated { + len: blob.len(), + needed: totalsize, + }); + } + Ok(Header { + totalsize, + off_dt_struct: word(8)?, + off_dt_strings: word(12)?, + off_mem_rsvmap: word(16)?, + boot_cpuid_phys: word(28)?, + size_dt_strings: word(32)?, + size_dt_struct: word(36)?, + }) +} + +/// Reads (address, size) pairs up to the all-zero terminator. +fn parse_reservations(data: &[u8], off: u32) -> Result, Error> { + const PAST_END: Error = Error::Malformed("memory reservation block runs past totalsize"); + let mut entries = Vec::new(); + let mut pos = off as usize; + loop { + let address = read_u64(data, pos).ok_or(PAST_END)?; + let size = read_u64(data, pos + 8).ok_or(PAST_END)?; + pos += RESERVE_ENTRY_LEN; + if address == 0 && size == 0 { + return Ok(entries); + } + entries.push((address, size)); + } +} + +fn parse_struct(structs: &[u8], strings: &[u8]) -> Result { + let mut stack: Vec = Vec::new(); + let mut root: Option = None; + let mut pos = 0usize; + loop { + let token = read_u32(structs, pos).ok_or(TRUNCATED_STRUCT)?; + pos += 4; + match token { + FDT_BEGIN_NODE => { + if stack.is_empty() && root.is_some() { + return Err(Error::Malformed("more than one root node")); + } + let (name, next) = read_padded_name(structs, pos)?; + pos = next; + stack.push(Node { + name, + props: Vec::new(), + children: Vec::new(), + }); + } + FDT_END_NODE => { + let node = stack + .pop() + .ok_or(Error::Malformed("FDT_END_NODE with no open node"))?; + match stack.last_mut() { + Some(parent) => parent.children.push(node), + None => root = Some(node), + } + } + FDT_PROP => { + let len = read_u32(structs, pos).ok_or(TRUNCATED_STRUCT)? as usize; + let name_off = read_u32(structs, pos + 4).ok_or(TRUNCATED_STRUCT)?; + pos += 8; + let end = pos.checked_add(len).ok_or(PROP_PAST_STRUCT)?; + let value = structs.get(pos..end).ok_or(PROP_PAST_STRUCT)?.to_vec(); + pos = align4(end); + if pos > structs.len() { + return Err(PROP_PAST_STRUCT); + } + let name = string_at(strings, name_off)?; + let parent = stack + .last_mut() + .ok_or(Error::Malformed("property outside any node"))?; + parent.props.push(Prop { name, value }); + } + FDT_NOP => {} + FDT_END => { + if !stack.is_empty() { + return Err(Error::Malformed("FDT_END inside an open node")); + } + return root.ok_or(Error::Malformed("no root node before FDT_END")); + } + _ => return Err(Error::Malformed("unknown structure token")), + } + } +} + +/// Reads a NUL-terminated node name at `at`; returns it with the next token offset. +fn read_padded_name(structs: &[u8], at: usize) -> Result<(Vec, usize), Error> { + let rest = structs + .get(at..) + .ok_or(Error::Malformed("node name starts outside structure block"))?; + let nul = rest + .iter() + .position(|&b| b == 0) + .ok_or(Error::Malformed("unterminated node name"))?; + let next = align4(at + nul + 1); + if next > structs.len() { + return Err(Error::Malformed( + "node name padding runs past structure block", + )); + } + Ok((rest[..nul].to_vec(), next)) +} + +fn string_at(strings: &[u8], at: u32) -> Result, Error> { + let rest = strings.get(at as usize..).ok_or(Error::Malformed( + "property name offset outside strings block", + ))?; + let nul = rest + .iter() + .position(|&b| b == 0) + .ok_or(Error::Malformed("unterminated property name"))?; + Ok(rest[..nul].to_vec()) +} + +/// Serializes `root` into a version 17 blob. Property names are interned into the +/// strings block, so each distinct name is stored once. +fn serialize( + root: &Node, + reservations: &[(u64, u64)], + boot_cpuid_phys: u32, +) -> Result, Error> { + let mut writer = Writer { + structs: Vec::new(), + strings: Vec::new(), + names: Vec::new(), + }; + writer.node(root)?; + push_u32(&mut writer.structs, FDT_END); + + let off_mem_rsvmap = HEADER_LEN; + let rsvmap_len = (reservations.len() + 1) * RESERVE_ENTRY_LEN; + let off_dt_struct = off_mem_rsvmap + rsvmap_len; + let off_dt_strings = off_dt_struct + writer.structs.len(); + let totalsize = off_dt_strings + writer.strings.len(); + + let to_u32 = + |value: usize| u32::try_from(value).map_err(|_| Error::Malformed("blob exceeds 4 GiB")); + let mut out = Vec::with_capacity(totalsize); + push_u32(&mut out, MAGIC); + push_u32(&mut out, to_u32(totalsize)?); + push_u32(&mut out, to_u32(off_dt_struct)?); + push_u32(&mut out, to_u32(off_dt_strings)?); + push_u32(&mut out, to_u32(off_mem_rsvmap)?); + push_u32(&mut out, VERSION); + push_u32(&mut out, LAST_COMP_VERSION); + push_u32(&mut out, boot_cpuid_phys); + push_u32(&mut out, to_u32(writer.strings.len())?); + push_u32(&mut out, to_u32(writer.structs.len())?); + for &(address, size) in reservations { + out.extend_from_slice(&address.to_be_bytes()); + out.extend_from_slice(&size.to_be_bytes()); + } + out.extend_from_slice(&[0; RESERVE_ENTRY_LEN]); + out.extend_from_slice(&writer.structs); + out.extend_from_slice(&writer.strings); + Ok(out) +} + +struct Writer { + structs: Vec, + strings: Vec, + names: Vec<(Vec, u32)>, +} + +impl Writer { + fn name_offset(&mut self, name: &[u8]) -> Result { + if let Some((_, offset)) = self.names.iter().find(|(n, _)| n.as_slice() == name) { + return Ok(*offset); + } + let offset = u32::try_from(self.strings.len()) + .map_err(|_| Error::Malformed("strings block exceeds 4 GiB"))?; + self.strings.extend_from_slice(name); + self.strings.push(0); + self.names.push((name.to_vec(), offset)); + Ok(offset) + } + + fn node(&mut self, node: &Node) -> Result<(), Error> { + push_u32(&mut self.structs, FDT_BEGIN_NODE); + self.structs.extend_from_slice(&node.name); + self.structs.push(0); + pad4(&mut self.structs); + for prop in &node.props { + let name_off = self.name_offset(&prop.name)?; + let len = u32::try_from(prop.value.len()) + .map_err(|_| Error::Malformed("property value exceeds 4 GiB"))?; + push_u32(&mut self.structs, FDT_PROP); + push_u32(&mut self.structs, len); + push_u32(&mut self.structs, name_off); + self.structs.extend_from_slice(&prop.value); + pad4(&mut self.structs); + } + for child in &node.children { + self.node(child)?; + } + push_u32(&mut self.structs, FDT_END_NODE); + Ok(()) + } +} + +fn block<'a>(data: &'a [u8], off: u32, size: u32, what: &'static str) -> Result<&'a [u8], Error> { + let start = off as usize; + let end = start + .checked_add(size as usize) + .ok_or(Error::Malformed(what))?; + data.get(start..end).ok_or(Error::Malformed(what)) +} + +fn read_u32(bytes: &[u8], at: usize) -> Option { + let end = at.checked_add(4)?; + let word: [u8; 4] = bytes.get(at..end)?.try_into().ok()?; + Some(u32::from_be_bytes(word)) +} + +fn read_u64(bytes: &[u8], at: usize) -> Option { + let end = at.checked_add(8)?; + let word: [u8; 8] = bytes.get(at..end)?.try_into().ok()?; + Some(u64::from_be_bytes(word)) +} + +fn align4(value: usize) -> usize { + (value + 3) & !3 +} + +fn push_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_be_bytes()); +} + +fn pad4(out: &mut Vec) { + while !out.len().is_multiple_of(4) { + out.push(0); + } +} + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + + fn prop(name: &str, value: &[u8]) -> Prop { + Prop { + name: name.as_bytes().to_vec(), + value: value.to_vec(), + } + } + + fn cstr(s: &str) -> Vec { + let mut v = s.as_bytes().to_vec(); + v.push(0); + v + } + + fn be32(words: &[u32]) -> Vec { + words.iter().flat_map(|w| w.to_be_bytes()).collect() + } + + fn node(name: &str, props: Vec, children: Vec) -> Node { + Node { + name: name.as_bytes().to_vec(), + props, + children, + } + } + + /// Root with a memory-controller (not a memory device), memory, chosen and a serial. + fn fixture_with_cells(address_cells: u32, size_cells: u32) -> Node { + let mut root = Node::named(b""); + root.props = vec![ + prop("#address-cells", &be32(&[address_cells])), + prop("#size-cells", &be32(&[size_cells])), + prop("compatible", &cstr("apple,j274")), + ]; + root.children = vec![ + node( + "memory-controller@20e000000", + vec![prop("compatible", &cstr("apple,memory-controller"))], + vec![], + ), + node( + "memory@800000000", + vec![ + prop("device_type", &cstr("memory")), + prop("reg", &be32(&[8, 0, 2, 0])), + ], + vec![], + ), + node( + "chosen", + vec![prop("stdout-path", &cstr("/serial@235200000"))], + vec![], + ), + node( + "serial@235200000", + vec![ + prop("compatible", &cstr("ns16550")), + prop("reg", &be32(&[2, 0x3520_0000, 0, 0x4000])), + ], + vec![], + ), + ]; + root + } + + fn fixture() -> Node { + fixture_with_cells(2, 2) + } + + fn blob_of(root: &Node) -> Vec { + serialize(root, &[], 0).unwrap() + } + + fn tree_of(blob: &[u8]) -> Node { + parse(blob).unwrap().root + } + + fn child<'a>(node: &'a Node, name: &str) -> &'a Node { + node.children + .iter() + .find(|c| c.name == name.as_bytes()) + .unwrap() + } + + fn prop_of<'a>(node: &'a Node, name: &str) -> &'a [u8] { + prop_value(node, name.as_bytes()).unwrap() + } + + fn set_u32(blob: &mut [u8], at: usize, value: u32) { + blob[at..at + 4].copy_from_slice(&value.to_be_bytes()); + } + + #[test] + fn round_trip_keeps_tree_order_and_reservations() { + let original = fixture(); + let reservations = [(0x4000_0000u64, 0x0010_0000u64)]; + let blob = serialize(&original, &reservations, 0).unwrap(); + let out = patch(&blob, None, None).unwrap(); + let parsed = parse(&out).unwrap(); + assert_eq!(parsed.root, original); + assert_eq!(parsed.reservations, reservations); + } + + #[test] + fn bootargs_is_appended_to_chosen_when_absent() { + let out = patch( + &blob_of(&fixture()), + Some("console=ttyS0 root=/dev/vda"), + None, + ) + .unwrap(); + let out = tree_of(&out); + let chosen = child(&out, "chosen"); + let names: Vec<&[u8]> = chosen.props.iter().map(|p| p.name.as_slice()).collect(); + assert_eq!(names, [&b"stdout-path"[..], &b"bootargs"[..]]); + assert_eq!( + prop_of(chosen, "bootargs"), + b"console=ttyS0 root=/dev/vda\0" + ); + } + + #[test] + fn bootargs_replaces_existing_property_in_place() { + let mut root = fixture(); + let chosen = root + .children + .iter_mut() + .find(|c| c.name == b"chosen") + .unwrap(); + chosen.props.insert(0, prop("bootargs", &cstr("old"))); + let out = tree_of(&patch(&blob_of(&root), Some("new args"), None).unwrap()); + let chosen = child(&out, "chosen"); + let names: Vec<&[u8]> = chosen.props.iter().map(|p| p.name.as_slice()).collect(); + assert_eq!(names, [&b"bootargs"[..], &b"stdout-path"[..]]); + assert_eq!(prop_of(chosen, "bootargs"), b"new args\0"); + } + + #[test] + fn chosen_is_created_at_root_when_absent() { + let mut root = fixture(); + root.children.retain(|c| c.name != b"chosen"); + let out = tree_of(&patch(&blob_of(&root), Some("quiet"), None).unwrap()); + assert_eq!(out.children.len(), root.children.len() + 1); + assert_eq!(&out.children[..root.children.len()], &root.children[..]); + let chosen = out.children.last().unwrap(); + assert_eq!(chosen.name, b"chosen"); + assert_eq!(chosen.props, vec![prop("bootargs", b"quiet\0")]); + } + + #[test] + fn memory_reg_uses_two_two_cells() { + let out = patch( + &blob_of(&fixture()), + None, + Some((0x1_4000_0000, 0x2_0000_0000)), + ) + .unwrap(); + let out = tree_of(&out); + let mem = child(&out, "memory@800000000"); + assert_eq!(prop_of(mem, "reg"), be32(&[1, 0x4000_0000, 2, 0])); + } + + #[test] + fn memory_reg_uses_one_one_cells() { + let out = patch( + &blob_of(&fixture_with_cells(1, 1)), + None, + Some((0x4000_0000, 0x0800_0000)), + ) + .unwrap(); + let out = tree_of(&out); + let mem = child(&out, "memory@800000000"); + assert_eq!(prop_of(mem, "reg"), be32(&[0x4000_0000, 0x0800_0000])); + } + + #[test] + fn patch_touches_only_chosen_and_memory_reg() { + let original = fixture(); + let out = tree_of( + &patch( + &blob_of(&original), + Some("quiet"), + Some((0x1_4000_0000, 0x2_0000_0000)), + ) + .unwrap(), + ); + assert_eq!(out.props, original.props); + assert_eq!(out.name, original.name); + assert_eq!( + child(&out, "memory-controller@20e000000"), + child(&original, "memory-controller@20e000000") + ); + assert_eq!( + child(&out, "serial@235200000"), + child(&original, "serial@235200000") + ); + assert_eq!(out.children.len(), original.children.len()); + assert_eq!( + prop_of(child(&out, "memory@800000000"), "device_type"), + b"memory\0" + ); + assert_eq!( + prop_of(child(&out, "chosen"), "stdout-path"), + b"/serial@235200000\0" + ); + } + + #[test] + fn memory_value_too_large_for_one_cell_is_rejected() { + let blob = blob_of(&fixture_with_cells(1, 1)); + assert_eq!( + patch(&blob, None, Some((1 << 32, 0x1000))), + Err(Error::ValueTooLarge { + value: 1 << 32, + cells: 1 + }) + ); + } + + #[test] + fn unsupported_cell_count_is_rejected() { + let blob = blob_of(&fixture_with_cells(3, 2)); + assert_eq!( + patch(&blob, None, Some((0x4000_0000, 0x1000))), + Err(Error::CellCount(3)) + ); + } + + #[test] + fn memory_patch_needs_a_memory_device_node() { + let mut root = fixture(); + root.children.retain(|c| !c.name.starts_with(b"memory@")); + assert_eq!( + patch(&blob_of(&root), None, Some((0x4000_0000, 0x1000))), + Err(Error::NoMemoryNode) + ); + } + + #[test] + fn bad_magic_is_rejected() { + let mut blob = blob_of(&fixture()); + blob[0] ^= 0xff; + assert!(matches!(patch(&blob, None, None), Err(Error::BadMagic(_)))); + } + + #[test] + fn truncated_blob_is_rejected() { + let blob = blob_of(&fixture()); + assert!(matches!( + patch(&blob[..blob.len() - 1], None, None), + Err(Error::Truncated { .. }) + )); + assert!(matches!( + patch(&blob[..HEADER_LEN - 1], None, None), + Err(Error::Truncated { .. }) + )); + } + + #[test] + fn offsets_past_end_are_rejected() { + let blob = blob_of(&fixture()); + let total = blob.len() as u32; + + let mut bad_struct = blob.clone(); + set_u32(&mut bad_struct, 8, total + 4); + assert!(matches!( + patch(&bad_struct, None, None), + Err(Error::Malformed(_)) + )); + + let mut bad_strings = blob; + set_u32(&mut bad_strings, 12, u32::MAX); + assert!(matches!( + patch(&bad_strings, None, None), + Err(Error::Malformed(_)) + )); + } +} diff --git a/src/emulator/src/system/m1.rs b/src/emulator/src/system/m1.rs new file mode 100644 index 0000000..b49cdec --- /dev/null +++ b/src/emulator/src/system/m1.rs @@ -0,0 +1,216 @@ +//! Apple M1 (t8103) SoC: the devices the Asahi Linux kernel's +//! `arch/arm64/boot/dts/apple/t8103.dtsi` describes, far enough to boot it. +//! +//! The addresses are the ones the M1 Mac mini's own device tree (`DeviceTree.j274ap` +//! in the macOS restore image) gives: arm-io's `ranges` places the SoC at +//! `0x2_0000_0000`, and the Linux nodes use the same absolute addresses. DRAM starts +//! at `0x8_0000_0000`. The kernel is booted with the device tree the caller supplies, +//! patched to describe the RAM this run provides. +use super::boot::{self, Error, Layout}; +use super::dtb; +use crate::devices::aic::{self, Aic}; +use crate::devices::pmgr::Pmgr; +use crate::devices::s5l_uart::{self, S5lUart}; +use crate::memory::GuestMemory; + +/// DRAM base (`memory@800000000` in the Asahi device tree). +pub const RAM_BASE: u64 = 0x8_0000_0000; +/// Default guest RAM: enough for the Linux kernel's own memory, well below the host's. +pub const DEFAULT_RAM_SIZE: usize = 1 << 30; +/// Interrupt controller (`aic: interrupt-controller@23b100000`). +pub const AIC_BASE: u64 = 0x2_3b10_0000; +/// Power-state register blocks (`power-management@…` with the `apple,pmgr` syscon): +/// the always-on block and the NUB block, as the device tree gives them. +pub const PMGR_BLOCKS: [(u64, u64); 2] = [(0x2_3b70_0000, 0x1_4000), (0x2_3d28_0000, 0x4000)]; +/// Console (`serial0: serial@235200000`), the `stdout-path` of the M1 device trees. +pub const UART_BASE: u64 = 0x2_3520_0000; +/// Hardware IRQ of the console (`AIC_IRQ 605` on `serial0`). +pub const UART_IRQ: u32 = 605; +/// Physical space below DRAM, where the device tree (an identity-mapped `simple-bus`) +/// places the SoC's devices. Devices this model does not implement live here. +pub const UNMODELED_BASE: u64 = 0; +pub const UNMODELED_SIZE: u64 = RAM_BASE; +/// `earlycon` without an address takes the device from `stdout-path`. +pub const DEFAULT_CMDLINE: &str = "earlycon nokaslr loglevel=8"; + +/// `MIDR_EL1` of an Icestorm core: implementer Apple (0x61), architecture from ID +/// registers (0xf), part 0x022. +pub const MIDR_EL1: u64 = 0x610f_0220; +/// `ID_AA64PFR0_EL1`: EL0 and EL1 in AArch64, no EL2 or EL3, and floating point and +/// Advanced SIMD implemented, as on the M1. +pub const ID_AA64PFR0_EL1: u64 = 0x11; +/// `ID_AA64MMFR0_EL1`: 4 KiB and 16 KiB translation granules (the Asahi kernel is built +/// for 16 KiB pages and refuses a CPU without one), no 64 KiB granule, 40-bit PA. +pub const ID_AA64MMFR0_EL1: u64 = (0xf << 24) | (1 << 20) | 2; + +/// The machine runs one vCPU. The device tree lists eight cores, but their +/// `cpu-release-addr` is zero, so Linux's spin-table code does not start them. +const CPUS: u32 = 1; + +/// Give `system` the identity of an M1 core. +pub fn identify(system: &mut crate::aarch64::cpu::System) { + system.midr_el1 = MIDR_EL1; + system.id_aa64pfr0_el1 = ID_AA64PFR0_EL1; + system.id_aa64mmfr0_el1 = ID_AA64MMFR0_EL1; +} + +/// The SoC devices on the bus: the AIC, the power-state blocks, the console, and the +/// line between the console and the AIC. +pub struct Soc { + aic: Aic, + pmgr: [Pmgr; PMGR_BLOCKS.len()], + uart: S5lUart, + unmodeled: u64, +} + +impl Soc { + /// `sink` receives every byte the kernel writes to the console. + pub fn new(sink: impl FnMut(u8) + Send + 'static) -> Self { + Self { + aic: Aic::new(CPUS), + pmgr: PMGR_BLOCKS.map(|(_, len)| Pmgr::new(len)), + uart: S5lUart::new(sink), + unmodeled: 0, + } + } + + /// Present the current device lines to the interrupt controller. Call before each + /// run of the CPU. + pub fn sync_lines(&mut self) { + self.aic.set_irq(UART_IRQ, self.uart.irq_asserted()); + } + + /// Whether the IRQ pin of CPU `cpu` is asserted. + pub fn irq_pending(&self, cpu: u32) -> bool { + self.aic.irq_pending(cpu) + } + + /// Accesses to arm-io devices this model does not implement: reads return zero and + /// writes are dropped, so a probe of such a device cannot fault the kernel. + pub fn unmodeled_accesses(&self) -> u64 { + self.unmodeled + } + + /// The power-state block and offset an access of `size` bytes at `address` falls in. + fn pmgr_window(address: u64, size: u8) -> Option<(usize, u64)> { + PMGR_BLOCKS + .iter() + .enumerate() + .find_map(|(block, &(base, len))| { + window(address, size, base, len).map(|offset| (block, offset)) + }) + } + + /// Read `size` bytes at guest-physical `address`, or `None` when no SoC device maps it. + pub fn read(&mut self, cpu: u32, address: u64, size: u8) -> Option { + if let Some(offset) = window(address, size, AIC_BASE, aic::LEN) { + return Some(self.aic.read(cpu, offset, size)); + } + if let Some((block, offset)) = Self::pmgr_window(address, size) { + return Some(self.pmgr[block].read(offset, size)); + } + if let Some(offset) = window(address, size, UART_BASE, s5l_uart::LEN) { + return Some(self.uart.read(offset, size)); + } + if window(address, size, UNMODELED_BASE, UNMODELED_SIZE).is_some() { + self.unmodeled += 1; + return Some(0); + } + None + } + + /// Write `size` bytes to guest-physical `address`; `false` when no SoC device maps it. + pub fn write(&mut self, cpu: u32, address: u64, size: u8, value: u64) -> bool { + if let Some(offset) = window(address, size, AIC_BASE, aic::LEN) { + self.aic.write(cpu, offset, size, value); + return true; + } + if let Some((block, offset)) = Self::pmgr_window(address, size) { + self.pmgr[block].write(offset, size, value); + return true; + } + if let Some(offset) = window(address, size, UART_BASE, s5l_uart::LEN) { + self.uart.write(offset, size, value); + return true; + } + if window(address, size, UNMODELED_BASE, UNMODELED_SIZE).is_some() { + self.unmodeled += 1; + return true; + } + false + } +} + +/// Offset of an access of `size` bytes at `address` within `[base, base + len)`. +/// An access that straddles the end of the window matches nothing. +fn window(address: u64, size: u8, base: u64, len: u64) -> Option { + let end = address.checked_add(u64::from(size))?; + (address >= base && end <= base + len).then(|| address - base) +} + +/// Place a Linux `Image` in RAM and the device tree after it, with the tree's +/// `/chosen/bootargs` set to `cmdline` (when non-empty) and its memory node describing +/// `ram_size` bytes at [`RAM_BASE`]. The kernel is entered with `x0` = the tree. +pub fn prepare( + memory: &mut impl GuestMemory, + image: &[u8], + device_tree: &[u8], + cmdline: &str, + ram_size: u64, +) -> Result { + let bootargs = (!cmdline.is_empty()).then_some(cmdline); + let tree = + dtb::patch(device_tree, bootargs, Some((RAM_BASE, ram_size))).map_err(Error::DeviceTree)?; + boot::place_with_tree(memory, image, &tree, RAM_BASE, ram_size) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn window_matches_only_accesses_entirely_inside_the_device() { + assert_eq!(window(AIC_BASE, 4, AIC_BASE, aic::LEN), Some(0)); + assert_eq!( + window(AIC_BASE + aic::LEN - 4, 4, AIC_BASE, aic::LEN), + Some(aic::LEN - 4) + ); + // Straddling the end, or wrapping, matches nothing. + assert_eq!(window(AIC_BASE + aic::LEN - 2, 4, AIC_BASE, aic::LEN), None); + assert_eq!(window(u64::MAX - 1, 4, AIC_BASE, aic::LEN), None); + } + + #[test] + fn soc_routes_by_address_and_declines_everything_else() { + let mut soc = Soc::new(|_| {}); + // The AIC identifies the reading CPU at WHOAMI. + assert_eq!(soc.read(0, AIC_BASE + 0x2000, 4), Some(0)); + // Above DRAM, where the SoC maps nothing, is not an M1 address. + assert_eq!(soc.read(0, RAM_BASE + (1 << 40), 4), None); + assert!(!soc.write(0, RAM_BASE + (1 << 40), 4, 0)); + assert_eq!(soc.unmodeled_accesses(), 0); + } + + #[test] + fn power_state_registers_of_each_pmgr_block_acknowledge_their_target() { + let mut soc = Soc::new(|_| {}); + for (base, _) in PMGR_BLOCKS { + // A domain's state register, as its driver polls it. + let state = base + 0x100; + assert_eq!(soc.read(0, state, 4), Some(0xff)); + assert!(soc.write(0, state, 4, 0x0)); + assert_eq!(soc.read(0, state, 4), Some(0x00)); + } + assert_eq!(soc.unmodeled_accesses(), 0); + } + + #[test] + fn unmodeled_arm_io_device_reads_zero_and_drops_writes_and_is_counted() { + let mut soc = Soc::new(|_| {}); + // The pin controller (`pinctrl@23e820000`) is probed at boot. + assert_eq!(soc.read(0, 0x2_3e82_0000, 4), Some(0)); + assert!(soc.write(0, 0x2_3e82_0000, 4, 0xffff_ffff)); + assert_eq!(soc.read(0, 0x2_3e82_0000, 4), Some(0)); + assert_eq!(soc.unmodeled_accesses(), 3); + } +} diff --git a/src/emulator/src/system/machine.rs b/src/emulator/src/system/machine.rs index 5abff51..bfebffa 100644 --- a/src/emulator/src/system/machine.rs +++ b/src/emulator/src/system/machine.rs @@ -44,6 +44,23 @@ pub trait DeviceIo { fn write_virtio(&mut self, offset: u64, size: u8, value: u64) { let _ = (offset, size, value); } + /// A device the board places anywhere in the address space (the M1 SoC's + /// interrupt controller and UART). `None` declines the access, which then goes + /// to the fixed windows below. + fn read_soc(&mut self, cpu_id: u32, address: u64, size: u8) -> Option { + let _ = (cpu_id, address, size); + None + } + /// Returns `false` when the board's devices do not claim the address. + fn write_soc(&mut self, cpu_id: u32, address: u64, size: u8, value: u64) -> bool { + let _ = (cpu_id, address, size, value); + false + } + /// Whether the fixed windows below (the QEMU `virt` UART, GIC and virtio) exist on + /// this board. A board that maps none of them makes accesses there fault. + fn has_fixed_windows(&self) -> bool { + true + } } impl DeviceIo for (&mut Pl011, &mut Gicv2) { @@ -167,6 +184,10 @@ pub struct Machine { /// Set when a GICv3 forwards PPI 27 (the virtual timer) as a group 0 FIQ: the /// priority it was given. `None` leaves the timer to the GICv2 path. pub vtimer_fiq: Option, + /// A FIQ source the board drives directly. The M1 interrupt controller presents + /// the virtual timer this way: it is level-high while the timer's own condition + /// holds, with no GIC-style priority gating. + pub fiq_line: bool, timer_fired: bool, pub stalled_at: u64, pub stalled: Option, @@ -201,6 +222,7 @@ impl Machine { watch_epoch: 0, irq_line: false, vtimer_fiq: None, + fiq_line: false, timer_fired: false, stalled_at: 0, stalled: None, @@ -269,7 +291,7 @@ impl Machine { if self.timer_edge() { return Ok(Exit::Timer); } - let fiq = self.fiq_asserted(); + let fiq = self.fiq_line || self.fiq_asserted(); self.cpu.system.isr_el1 = u64::from(self.irq_line) << 7 | u64::from(fiq) << 6; let kind = if fiq && self.cpu.system.daif & 1 == 0 { Some(exception::Kind::Fiq) @@ -384,14 +406,32 @@ impl Machine { self.cpu.trap = Trap::None; exception::eret(&mut self.cpu); } - Trap::Svc | Trap::Brk => { - let ec = if self.cpu.trap == Trap::Svc { 0x15 } else { 0 }; + Trap::Svc => { + self.cpu.trap = Trap::None; + let pc = self.cpu.pc; + exception::take( + &mut self.cpu, + exception::Reason { + ec: 0x15, + status: 0, + instruction: true, + ..Default::default() + }, + pc, + ); + } + Trap::Brk => { self.cpu.trap = Trap::None; let pc = self.cpu.pc; + let immediate = self.cpu.address as u32; + // A breakpoint's syndrome is EC 0x3c with its immediate in ISS. exception::take( &mut self.cpu, exception::Reason { - ec, + ec: 0x3c, + status: 0, + instruction: true, + iss: immediate, ..Default::default() }, pc, @@ -407,12 +447,12 @@ impl Machine { ec: 0, instruction: true, status: 0, + iss: 0, }, pc, ); } Trap::Isb => { - self.tlb.purge_faults(); self.cpu.trap = Trap::None; } Trap::Tlbi => { @@ -613,6 +653,15 @@ impl Machine { } } fn abort(&mut self, address: u64, permission: bool, instruction: bool, write: bool) { + let status = match (permission, instruction) { + (false, _) => exception::status::TRANSLATION, + (true, false) => exception::status::PERMISSION, + (true, true) => exception::status::PERMISSION_FETCH, + }; + self.abort_with_status(address, status, instruction, write); + } + /// A synchronous abort with fault status `status`. + fn abort_with_status(&mut self, address: u64, status: u8, instruction: bool, write: bool) { let ec = if instruction { if self.cpu.system.el == 1 { 0x21 } else { 0x20 } } else if self.cpu.system.el == 1 { @@ -620,11 +669,6 @@ impl Machine { } else { 0x24 }; - let status = if permission { - if instruction { 13 } else { 12 } - } else { - 4 - }; self.cpu.trap = Trap::None; if !instruction { self.cpu.pc = self.cpu.pc.wrapping_sub(4); @@ -636,6 +680,7 @@ impl Machine { ec, instruction, status, + iss: 0, }, address, ); @@ -963,7 +1008,7 @@ impl Machine { }; if matches!(result, Err(MemoryError::AccessFault { .. })) { self.unmapped += 1; - self.abort(address, false, false, !load); + self.abort_with_status(address, exception::status::SYNC_EXTERNAL, false, !load); return Ok(()); } result @@ -973,9 +1018,20 @@ impl Machine { Err(MemoryError::AccessFault { .. }) => { self.device_access = true; let end = physical.checked_add(width as u64); - let window = - |base, length| physical >= base && end.is_some_and(|end| end <= base + length); - let read = if window(fdt::UART_BASE, crate::devices::pl011::LEN) { + let fixed = devices.has_fixed_windows(); + let window = |base, length| { + fixed && physical >= base && end.is_some_and(|end| end <= base + length) + }; + let soc = if load { + devices.read_soc(self.cpu_id, physical, width).map(Some) + } else { + devices + .write_soc(self.cpu_id, physical, width, value) + .then_some(None) + }; + let read = if let Some(read) = soc { + read + } else if window(fdt::UART_BASE, crate::devices::pl011::LEN) { if load { Some(devices.read_uart(physical - fdt::UART_BASE, width)) } else { @@ -1026,7 +1082,7 @@ impl Machine { } } else { self.unmapped += 1; - self.abort(address, false, false, !load); + self.abort_with_status(address, exception::status::SYNC_EXTERNAL, false, !load); return Ok(()); }; if let Some(value) = read { @@ -1273,6 +1329,39 @@ mod tests { Exit::Psci { .. } )); } + + #[test] + fn brk_takes_an_el1_breakpoint_exception_with_its_immediate_in_the_syndrome() { + let mut machine = machine(); + machine.cpu.system.el = 1; + machine.cpu.system.spsel = true; + machine.cpu.system.vbar_el1 = 0x40000400; + // `brk #0x800`, the immediate the kernel's `BUG()` uses. + machine + .memory + .write(0x40000000, &0xd4210000u32.to_le_bytes()) + .unwrap(); + // The vector for a synchronous exception from EL1 on SP_EL1: `VBAR + 0x200`. + machine + .memory + .write(0x40000600, &HVC.to_le_bytes()) + .unwrap(); + machine.cpu.pc = 0x40000000; + let mut serial = Pl011::new(|_| {}); + let mut gic = Gicv2::default(); + assert!(matches!( + machine.run(&mut serial, &mut gic).unwrap(), + Exit::Psci { .. } + )); + let esr = machine.cpu.system.esr_el1; + assert_eq!(esr >> 26, 0x3c, "EC is a breakpoint"); + assert_eq!((esr >> 25) & 1, 1, "IL: a 32-bit instruction"); + assert_eq!(esr & 0xffff, 0x800, "ISS is the immediate"); + assert_eq!( + machine.cpu.system.elr_el1, 0x40000000, + "ELR is the BRK itself" + ); + } const LDXR_X1_X0: u32 = 0xc85f7c01; const STXR_W2_X3_X0: u32 = 0xc8027c03; const HVC: u32 = 0xd4000002; diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index 2b4740a..c3da1b0 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -4,8 +4,10 @@ pub mod afdt; pub mod boot; +pub mod dtb; pub mod esr; pub mod fdt; +pub mod m1; pub mod machine; pub mod psci; pub mod smp; @@ -92,6 +94,8 @@ pub struct BootReport { pub executed_blocks: u64, pub timer_interrupts: u64, pub unmapped_accesses: u64, + /// Accesses to arm-io devices the M1 model does not implement (read as zero). + pub unmodeled_device_accesses: u64, pub stalled_at: Option, pub instruction: Option, pub last_fault_walk: Vec, @@ -169,6 +173,11 @@ impl core::fmt::Display for BootReport { for entry in &self.trace { writeln!(f, " pc {:x} sp {:x}", entry.pc, entry.sp)?; } + writeln!( + f, + "unmodeled device accesses {}", + self.unmodeled_device_accesses + )?; writeln!( f, "exits {}, blocks compiled {}, blocks executed {}, timer interrupts {}, unmapped accesses {}, elapsed {:.3}s", @@ -189,6 +198,18 @@ pub struct SystemConfig> { pub ram_size: usize, pub initrd: Option>, pub disk: Option, + pub board: Board, +} + +/// The machine a boot builds around the kernel. +#[derive(Debug, Clone, Default)] +pub enum Board { + /// QEMU `virt`-style board: a GICv2 for Linux, a GICv3 for XNU, a PL011 console. + #[default] + Virt, + /// Apple M1 (t8103) SoC ([`m1`]): an AIC and an S5L console. Linux only. The + /// caller supplies the device tree, such as the Asahi kernel's `t8103-j274.dtb`. + AppleM1 { device_tree: Vec }, } impl Default for SystemConfig { @@ -200,6 +221,7 @@ impl Default for SystemConfig { ram_size: RAM_SIZE, initrd: None, disk: None, + board: Board::Virt, } } } @@ -207,10 +229,12 @@ impl Default for SystemConfig { /// The interrupt controller a boot configures: the GICv2 a Linux device tree /// describes, or the GICv3 XNU's platform expects. Shared peripherals (UART, virtio) /// are wired to the GICv2 only; the GICv3 carries the virtual timer, which the -/// machine evaluates live from the timer registers. +/// machine evaluates live from the timer registers. The M1 board has no GIC: its +/// AIC sits in [`m1::Soc`] and presents the virtual timer as a FIQ. enum Intc { V2(Gicv2), V3(Gicv3), + Apple, } impl Intc { fn raise(&mut self, intid: u32) { @@ -231,35 +255,51 @@ impl Intc { fn signalled(&self, cpu: u32) -> bool { match self { Self::V2(gic) => gic.signalled(cpu), - Self::V3(_) => false, + Self::V3(_) | Self::Apple => false, } } /// See [`machine::Machine::vtimer_fiq`]. fn vtimer_fiq(&self) -> Option { match self { - Self::V2(_) => None, + Self::V2(_) | Self::Apple => None, Self::V3(gic) => gic.vtimer_fiq_priority(), } } + /// Whether the CPU sees a FIQ from the virtual timer, which is due at `due`. The + /// M1 AIC presents the timer's own condition with no further gating. + fn timer_fiq(&self, due: bool) -> bool { + matches!(self, Self::Apple) && due + } } +/// The bus of one board. The fixed windows (QEMU `virt`'s UART, GIC and virtio) exist +/// only on the virt board; on the M1 board `soc` answers instead, and the GIC accessors +/// below are never reached with [`Intc::Apple`]. struct SystemDevices<'a, V> { - serial: &'a mut Pl011, + serial: Option<&'a mut Pl011>, gic: &'a mut Intc, virtio: Option<&'a mut V>, + soc: Option<&'a mut m1::Soc>, } impl machine::DeviceIo for SystemDevices<'_, V> { fn read_uart(&mut self, offset: u64, size: u8) -> u64 { - self.serial.read(offset, size) + self.serial + .as_mut() + .expect("the fixed UART window exists only on the virt board, which has a PL011") + .read(offset, size) } fn write_uart(&mut self, offset: u64, size: u8, value: u64) { - self.serial.write(offset, size, value); + self.serial + .as_mut() + .expect("the fixed UART window exists only on the virt board, which has a PL011") + .write(offset, size, value); } fn read_gic_distributor(&mut self, cpu_id: u32, offset: u64, size: u8) -> u64 { match self.gic { Intc::V2(gic) => gic.read_distributor_for(cpu_id, offset, size), Intc::V3(gic) => gic.read_distributor(offset, size), + Intc::Apple => 0, } } fn write_gic_distributor(&mut self, cpu_id: u32, offset: u64, size: u8, value: u64) { @@ -268,13 +308,14 @@ impl machine::DeviceIo for SystemDevices<'_, V> { gic.write_distributor_for(cpu_id, offset, size, value); } Intc::V3(gic) => gic.write_distributor(offset, size, value), + Intc::Apple => {} } } fn read_gic_cpu(&mut self, cpu_id: u32, offset: u64, size: u8) -> u64 { match self.gic { Intc::V2(gic) => gic.read_cpu_for(cpu_id, offset, size), // The GICv3 CPU interface is the `ICC_*` system registers. - Intc::V3(_) => 0, + Intc::V3(_) | Intc::Apple => 0, } } fn write_gic_cpu(&mut self, cpu_id: u32, offset: u64, size: u8, value: u64) { @@ -285,7 +326,7 @@ impl machine::DeviceIo for SystemDevices<'_, V> { fn read_gic_redistributor(&mut self, offset: u64, size: u8) -> u64 { match self.gic { Intc::V3(gic) => gic.read_redistributor(offset, size), - Intc::V2(_) => 0, + Intc::V2(_) | Intc::Apple => 0, } } fn write_gic_redistributor(&mut self, offset: u64, size: u8, value: u64) { @@ -305,8 +346,18 @@ impl machine::DeviceIo for SystemDevices<'_, V> { v.write(offset, size, value); } } + fn read_soc(&mut self, cpu_id: u32, address: u64, size: u8) -> Option { + self.soc.as_mut()?.read(cpu_id, address, size) + } + fn write_soc(&mut self, cpu_id: u32, address: u64, size: u8, value: u64) -> bool { + self.soc + .as_mut() + .is_some_and(|soc| soc.write(cpu_id, address, size, value)) + } + fn has_fixed_windows(&self) -> bool { + self.soc.is_none() + } } - /// Boot a raw Linux Image with live serial output on stdout and a bounded wall-clock run. /// Execution faults are returned in the report with the full guest state, not discarded. pub fn boot(image: &[u8], timeout: Duration, cmdline: &str) -> Result { @@ -357,6 +408,7 @@ pub fn boot_with_options( ram_size: RAM_SIZE, initrd: None, disk: None, + board: Board::Virt, }, sink, ) @@ -375,26 +427,48 @@ pub fn boot_system( }; let timeout = config.timeout; let idle_mode = config.idle_mode; - let region = Region::ram(RAM_BASE, vec![0; ram_size]).map_err(Error::Memory)?; + let apple = matches!(config.board, Board::AppleM1 { .. }); + let ram_base = if apple { m1::RAM_BASE } else { RAM_BASE }; + let region = Region::ram(ram_base, vec![0; ram_size]).map_err(Error::Memory)?; let memory = PhysicalMemory::new(vec![region]).map_err(Error::Memory)?; let mut shared_mem = SharedMemory::new(memory); let has_virtio = config.disk.is_some(); let is_xnu = xnu::is_macho(image); - let layout = if is_xnu { - if config.initrd.is_some() || has_virtio { - return Err(Error::Boot(boot::Error::Unsupported( - "XNU boot has no initrd or virtio disk support", - ))); + let layout = match config.board { + Board::AppleM1 { device_tree } => { + if is_xnu { + return Err(Error::Boot(boot::Error::Unsupported( + "XNU boot on the Apple M1 SoC is not supported yet", + ))); + } + if has_virtio || config.initrd.is_some() { + return Err(Error::Boot(boot::Error::Unsupported( + "the Apple M1 SoC has no initrd or virtio disk support yet", + ))); + } + m1::prepare( + &mut shared_mem, + image, + &device_tree, + &config.cmdline, + ram_size as u64, + ) } - xnu::prepare_boot( - &mut shared_mem, - image, - &config.cmdline, - RAM_BASE, - ram_size as u64, - ) - } else { - boot::prepare_boot( + Board::Virt if is_xnu => { + if config.initrd.is_some() || has_virtio { + return Err(Error::Boot(boot::Error::Unsupported( + "XNU boot has no initrd or virtio disk support", + ))); + } + xnu::prepare_boot( + &mut shared_mem, + image, + &config.cmdline, + RAM_BASE, + ram_size as u64, + ) + } + Board::Virt => boot::prepare_boot( &mut shared_mem, image, config.initrd.as_deref(), @@ -403,22 +477,28 @@ pub fn boot_system( ram_size as u64, 1, has_virtio, - ) + ), } .map_err(Error::Boot)?; let console = Arc::new(Mutex::new(Vec::new())); let serial_console = console.clone(); - let uart_line = Arc::new(AtomicBool::new(false)); - let serial_line = uart_line.clone(); - let mut serial = Pl011::new(move |byte| { + let emit = move |byte: u8| { serial_console.lock().push(byte); sink(byte); - }) - .with_line(move |level| serial_line.store(level, Ordering::Release)); - let mut gic = if is_xnu { - Intc::V3(Gicv3::default()) + }; + let uart_line = Arc::new(AtomicBool::new(false)); + let serial_line = uart_line.clone(); + let (mut serial, mut soc, mut gic) = if apple { + (None, Some(m1::Soc::new(emit)), Intc::Apple) } else { - Intc::V2(Gicv2::default()) + let serial = + Pl011::new(emit).with_line(move |level| serial_line.store(level, Ordering::Release)); + let gic = if is_xnu { + Intc::V3(Gicv3::default()) + } else { + Intc::V2(Gicv2::default()) + }; + (Some(serial), None, gic) }; let virtio_line = Arc::new(AtomicBool::new(false)); let virtio_notify = virtio_line.clone(); @@ -427,6 +507,9 @@ pub fn boot_system( .with_line(move |level| virtio_notify.store(level, Ordering::Release)) }); let mut machine = machine::Machine::new(shared_mem); + if apple { + m1::identify(&mut machine.cpu.system); + } machine.cpu.pc = layout.entry; machine.cpu.x[0] = layout.boot_info; machine.cpu.x[1..4].fill(0); @@ -437,22 +520,32 @@ pub fn boot_system( if started.elapsed() >= timeout { break StopReason::Timeout; } - if uart_line.load(Ordering::Acquire) { - gic.raise(fdt::UART_INTID); - } else { - gic.lower(fdt::UART_INTID); - } - if virtio_line.load(Ordering::Acquire) { - gic.raise(fdt::VIRTIO_INTID); - } else { - gic.lower(fdt::VIRTIO_INTID); + match soc.as_mut() { + Some(soc) => soc.sync_lines(), + None => { + if uart_line.load(Ordering::Acquire) { + gic.raise(fdt::UART_INTID); + } else { + gic.lower(fdt::UART_INTID); + } + if virtio_line.load(Ordering::Acquire) { + gic.raise(fdt::VIRTIO_INTID); + } else { + gic.lower(fdt::VIRTIO_INTID); + } + } } - machine.irq_line = gic.signalled(0); + machine.irq_line = match &soc { + Some(soc) => soc.irq_pending(0), + None => gic.signalled(0), + }; + machine.fiq_line = gic.timer_fiq(machine.timer_due()); machine.vtimer_fiq = gic.vtimer_fiq(); let mut dev = SystemDevices { - serial: &mut serial, + serial: serial.as_mut(), gic: &mut gic, virtio: virtio.as_mut(), + soc: soc.as_mut(), }; match machine.run_with_devices(&mut dev) { Ok(machine::Exit::Timer) => { @@ -522,7 +615,7 @@ pub fn boot_system( if !is_xnu && machine.cpu.system.sctlr_el1 & 1 == 0 && sp != 0 - && !(RAM_BASE..=RAM_BASE + ram_size as u64).contains(&sp) + && !(ram_base..=ram_base + ram_size as u64).contains(&sp) { break StopReason::LostStack; } @@ -544,6 +637,7 @@ pub fn boot_system( executed_blocks: machine.blocks_run, timer_interrupts, unmapped_accesses: machine.unmapped, + unmodeled_device_accesses: soc.as_ref().map_or(0, |soc| soc.unmodeled_accesses()), stalled_at, instruction, last_fault_walk, @@ -585,41 +679,58 @@ fn instruction_diagnostic( } last } +/// The descriptors a translation of `address` reads, top level first, for the granule +/// and VA size the TCR selects. It follows `translate::walk` step for step, so the +/// report shows what the translation saw, for any granule. fn walk_diagnostic(cpu: &Cpu, memory: &impl GuestMemory, address: u64) -> Vec { let mut entries = Vec::new(); if address == 0 || cpu.system.sctlr_el1 & 1 == 0 { return entries; } - let upper = address >> 63 != 0; - let size = if upper { - (cpu.system.tcr_el1 >> 16) & 63 - } else { - cpu.system.tcr_el1 & 63 - }; - let granule = if upper { - (cpu.system.tcr_el1 >> 30) & 3 + let tcr = cpu.system.tcr_el1; + let (granule, va_bits, root) = if address >> 63 != 0 { + let granule = [0u32, 14, 12, 16][((tcr >> 30) & 3) as usize]; + let size = ((tcr >> 16) & 63) as u32; + (granule, 64 - size, cpu.system.ttbr1_el1) } else { - (cpu.system.tcr_el1 >> 14) & 3 + let granule = [12u32, 16, 14, 0][((tcr >> 14) & 3) as usize]; + let size = (tcr & 63) as u32; + (granule, 64 - size, cpu.system.ttbr0_el1) }; - if (!upper && granule != 0) || (upper && granule != 2) { + if granule == 0 || va_bits > 48 || va_bits <= granule { return entries; } - let bits = 64u64.saturating_sub(size); - let levels = bits.saturating_sub(12).div_ceil(9).clamp(1, 4) as u8; - let mut table = if upper { - cpu.system.ttbr1_el1 + let stride = granule - 3; + let levels = (va_bits - granule).div_ceil(stride); + if levels > 4 { + return entries; + } + let top_bits = va_bits - (granule + stride * (levels - 1)); + let root_alignment = if granule == 12 { + granule } else { - cpu.system.ttbr0_el1 - } & 0x0000_ffff_ffff_f000; - for level in 4 - levels..4 { - let index = (address >> (39 - 9 * level)) & 0x1ff; + (top_bits + 3).max(6) + }; + let address_mask = 0x0000_ffff_ffff_ffff & !((1u64 << granule) - 1); + let mut table = root & 0x0000_ffff_ffff_ffff & !((1u64 << root_alignment) - 1); + for depth in 0..levels { + // `translate::walk` counts levels from the leaf; the report counts from the top. + let from_leaf = levels - 1 - depth; + let shift = granule + stride * from_leaf; + let index_bits = if granule != 12 && depth == 0 { + top_bits + } else { + stride + }; + let index = (address >> shift) & ((1u64 << index_bits) - 1); + let at = table.wrapping_add(index * 8); let mut bytes = [0; 8]; let descriptor = memory - .read(table + index * 8, &mut bytes) + .read(at, &mut bytes) .ok() .map(|_| u64::from_le_bytes(bytes)); entries.push(WalkEntry { - level, + level: (4 - levels + depth) as u8, table, index, descriptor, @@ -627,10 +738,10 @@ fn walk_diagnostic(cpu: &Cpu, memory: &impl GuestMemory, address: u64) -> Vec Date: Wed, 7 Oct 2026 20:21:05 -0700 Subject: [PATCH 02/13] feat(emulator): add mh_filset support to xnu loader --- README.md | 5 +++-- src/emulator/src/aarch64/decode.rs | 3 +++ src/emulator/src/system/mod.rs | 13 ++++++----- src/emulator/src/system/xnu.rs | 35 ++++++++++++++++++++++++++---- 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 3399e49..68bbd32 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,9 @@ at `0x2_3520_0000`. The machine is one vCPU with the M1 Icestorm identity The Linux kernel boots through the AIC and the S5L console, probes its power domains, and panics on the missing root filesystem (`VFS: Unable to mount root fs`), which is the expected result without one. `--initrd`, `--disk` and `--smp` above 1 are rejected -on this board, and a Mach-O kernel is refused: XNU needs the SoC's own ADT and -firmware interfaces, which are not modelled. +on this board. A kernelcache from the restore image (`MH_FILESET`) loads, with its ADT +as `--dtb`, and runs its first blocks; it does not get past the early boot CPU +table yet, and it needs PAuth and chained fixups before it can go further. Not modelled: any arm-io device other than the AIC, the UART and the power-state blocks. Those addresses read as zero and writes are dropped, and the boot report counts diff --git a/src/emulator/src/aarch64/decode.rs b/src/emulator/src/aarch64/decode.rs index 1f445dc..71dcd15 100644 --- a/src/emulator/src/aarch64/decode.rs +++ b/src/emulator/src/aarch64/decode.rs @@ -1124,6 +1124,9 @@ fn apple_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option Apple(19), (7, 0, 4) => Apple(20), (7, 6, 4) => AppleZero, + // The S3_5_c15_cN_0 state registers early init reads, none named in the XNU + // source tree. Read as zero until their roles are identified. + (5, 2..=7, 0) => AppleZero, _ => return None, }) } diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index c3da1b0..e13c5d1 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -435,12 +435,15 @@ pub fn boot_system( let has_virtio = config.disk.is_some(); let is_xnu = xnu::is_macho(image); let layout = match config.board { + Board::AppleM1 { device_tree } if is_xnu => xnu::prepare_boot_with_tree( + &mut shared_mem, + image, + &config.cmdline, + &device_tree, + ram_base, + ram_size as u64, + ), Board::AppleM1 { device_tree } => { - if is_xnu { - return Err(Error::Boot(boot::Error::Unsupported( - "XNU boot on the Apple M1 SoC is not supported yet", - ))); - } if has_virtio || config.initrd.is_some() { return Err(Error::Boot(boot::Error::Unsupported( "the Apple M1 SoC has no initrd or virtio disk support yet", diff --git a/src/emulator/src/system/xnu.rs b/src/emulator/src/system/xnu.rs index f1a7115..4ee834a 100644 --- a/src/emulator/src/system/xnu.rs +++ b/src/emulator/src/system/xnu.rs @@ -15,6 +15,9 @@ use crate::memory::GuestMemory; const MH_MAGIC_64: u32 = 0xfeed_facf; const CPU_TYPE_ARM64: u32 = 0x0100_000c; const MH_EXECUTE: u32 = 2; +/// A kernel collection: one top-level header whose load commands list every segment of +/// every component, with `LC_FILESET_ENTRY` naming each component's own Mach-O header. +const MH_FILESET: u32 = 12; const LC_UNIXTHREAD: u32 = 0x5; const LC_SEGMENT_64: u32 = 0x19; const LC_MAIN: u32 = 0x8000_0028; @@ -85,8 +88,8 @@ pub fn parse(image: &[u8]) -> Result { if word(4) != CPU_TYPE_ARM64 { return Err(bad("not an arm64 Mach-O")); } - if word(12) != MH_EXECUTE { - return Err(bad("not an executable Mach-O")); + if word(12) != MH_EXECUTE && word(12) != MH_FILESET { + return Err(bad("not an executable or fileset Mach-O")); } let commands_end = HEADER_SIZE .checked_add(word(20) as usize) @@ -205,6 +208,20 @@ pub fn prepare_boot( cmdline: &str, ram_base: u64, ram_size: u64, +) -> Result { + let tree = super::afdt::build(ram_base, ram_size, cmdline); + prepare_boot_with_tree(memory, image, cmdline, &tree, ram_base, ram_size) +} + +/// [`prepare_boot`] with the device tree the caller supplies, such as the platform's +/// own ADT for the board. +pub fn prepare_boot_with_tree( + memory: &mut impl GuestMemory, + image: &[u8], + cmdline: &str, + tree: &[u8], + ram_base: u64, + ram_size: u64, ) -> Result { let kernel = parse(image)?; let ram_end = ram_base.checked_add(ram_size).ok_or(Error::NoRoom)?; @@ -235,7 +252,6 @@ pub fn prepare_boot( } let device_tree = align(kernel_end, PAGE)?; - let tree = super::afdt::build(ram_base, ram_size, cmdline); let args_at = align( device_tree .checked_add(tree.len() as u64) @@ -272,7 +288,7 @@ pub fn prepare_boot( .write(physical(segment.vmaddr)?, data) .map_err(Error::Memory)?; } - memory.write(device_tree, &tree).map_err(Error::Memory)?; + memory.write(device_tree, tree).map_err(Error::Memory)?; memory.write(args_at, &args).map_err(Error::Memory)?; Ok(Layout { entry, @@ -443,4 +459,15 @@ mod tests { Err(Error::Unsupported(_)) )); } + + #[test] + fn a_fileset_kernel_parses_like_an_executable_with_the_same_segments() { + let executable = image(Entry::Main(0x800)); + let mut fileset = executable.clone(); + fileset[12..16].copy_from_slice(&MH_FILESET.to_le_bytes()); + assert_eq!(parse(&fileset).unwrap(), parse(&executable).unwrap()); + let mut other = executable; + other[12..16].copy_from_slice(&1u32.to_le_bytes()); + assert!(matches!(parse(&other), Err(Error::BadMachO(_)))); + } } From a491db0b5a9596a7edd9a8e91c0dc52633eacc92 Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Wed, 7 Oct 2026 20:31:05 -0700 Subject: [PATCH 03/13] feat(emulator): handle pointer authentication --- src/emulator/src/aarch64/compile.rs | 2 + src/emulator/src/aarch64/cpu.rs | 6 + src/emulator/src/aarch64/decode.rs | 31 + src/emulator/src/aarch64/host.rs | 8 +- src/emulator/src/aarch64/identification.rs | 4 +- src/emulator/src/aarch64/mod.rs | 1 + src/emulator/src/aarch64/pauth.rs | 865 +++++++++++++++++++++ src/emulator/src/system/m1.rs | 4 + 8 files changed, 916 insertions(+), 5 deletions(-) create mode 100644 src/emulator/src/aarch64/pauth.rs diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index 21cab61..0858b93 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -2471,6 +2471,8 @@ fn system_offset(r: SystemRegister) -> Option { MidrEl1 => offset_of!(CpuSystem, midr_el1), IdAa64pfr0El1 => offset_of!(CpuSystem, id_aa64pfr0_el1), IdAa64mmfr0El1 => offset_of!(CpuSystem, id_aa64mmfr0_el1), + PacKey(slot) => offset_of!(CpuSystem, pac_keys) + 8 * usize::from(slot), + IdAa64isar1El1 => offset_of!(CpuSystem, id_aa64isar1_el1), _ => return None, }) } diff --git a/src/emulator/src/aarch64/cpu.rs b/src/emulator/src/aarch64/cpu.rs index a2d1b7b..802f7b6 100644 --- a/src/emulator/src/aarch64/cpu.rs +++ b/src/emulator/src/aarch64/cpu.rs @@ -206,6 +206,10 @@ pub struct System { pub midr_el1: u64, pub id_aa64pfr0_el1: u64, pub id_aa64mmfr0_el1: u64, + /// PAuth keys, low word first: APIA, APIB, APDA, APDB, APGA. + pub pac_keys: [u64; 10], + /// `ID_AA64ISAR1_EL1`: which pointer authentication algorithms the CPU advertises. + pub id_aa64isar1_el1: u64, } impl Default for System { fn default() -> Self { @@ -254,6 +258,8 @@ impl Default for System { midr_el1: crate::aarch64::identification::DEFAULT_MIDR_EL1, id_aa64pfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64PFR0_EL1, id_aa64mmfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64MMFR0_EL1, + pac_keys: [0; 10], + id_aa64isar1_el1: crate::aarch64::identification::DEFAULT_ID_AA64ISAR1_EL1, } } } diff --git a/src/emulator/src/aarch64/decode.rs b/src/emulator/src/aarch64/decode.rs index 71dcd15..530787f 100644 --- a/src/emulator/src/aarch64/decode.rs +++ b/src/emulator/src/aarch64/decode.rs @@ -606,6 +606,8 @@ pub enum SystemRegister { /// Apple IMP-DEF configuration register the guest reads back, stored in /// `System::apple[slot]`. Apple(u8), + /// PAuth key register `APxAKey{Lo,Hi}_EL1`, stored in `System::pac_keys[slot]`. + PacKey(u8), } impl SystemRegister { pub fn read_only(self) -> bool { @@ -1131,6 +1133,28 @@ fn apple_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option Option { + if op0 != 3 || op1 != 0 || crn != 2 { + return None; + } + let slot = match (crm, op2) { + (1, 0) => 0, + (1, 1) => 1, + (1, 2) => 2, + (1, 3) => 3, + (2, 0) => 4, + (2, 1) => 5, + (2, 2) => 6, + (2, 3) => 7, + (3, 0) => 8, + (3, 1) => 9, + _ => return None, + }; + Some(SystemRegister::PacKey(slot)) +} + fn system_register( op0: u32, op1: u32, @@ -1145,6 +1169,9 @@ fn system_register( if let Some(register) = apple_register(op0, op1, crn, crm, op2) { return Ok(register); } + if let Some(register) = pac_key_register(op0, op1, crn, crm, op2) { + return Ok(register); + } if op1 == 0 && crn == 4 && op2 == 5 { return Ok(SpSelSet); } @@ -1237,6 +1264,10 @@ fn system_register( pub fn decode(word: u32) -> Result { use DecodeError::UnsupportedInstruction; use Instruction::*; + // PAuth instructions, hint-space forms included, run on the host. + if super::pauth::is_supported(word) { + return Ok(Host(word)); + } let width = width(word); let rn = reg(word, 5); let rm = reg(word, 16); diff --git a/src/emulator/src/aarch64/host.rs b/src/emulator/src/aarch64/host.rs index e884992..3946e30 100644 --- a/src/emulator/src/aarch64/host.rs +++ b/src/emulator/src/aarch64/host.rs @@ -2,18 +2,20 @@ //! and traps; the machine (or user mode) then runs the word on the host here. //! Floating point and the wider integer SIMD families live in [`fp`] and [`simd`]. -use super::{Cpu, fp, simd}; +use super::{Cpu, fp, pauth, simd}; /// Whether [`run`] implements `word`. pub fn is_supported(word: u32) -> bool { - fp::is_supported(word) || simd::is_supported(word) + fp::is_supported(word) || simd::is_supported(word) || pauth::is_supported(word) } /// Run `word` on `cpu`. `word` must satisfy [`is_supported`]. pub fn run(cpu: &mut Cpu, word: u32) { if fp::is_supported(word) { fp::run(cpu, word) - } else { + } else if simd::is_supported(word) { simd::run(cpu, word) + } else { + pauth::run(cpu, word) } } diff --git a/src/emulator/src/aarch64/identification.rs b/src/emulator/src/aarch64/identification.rs index 595e76b..dec538f 100644 --- a/src/emulator/src/aarch64/identification.rs +++ b/src/emulator/src/aarch64/identification.rs @@ -21,6 +21,7 @@ pub const CCSIDR_L2: u64 = ccsidr(4, 2048, 8); pub const DEFAULT_MIDR_EL1: u64 = 0x410f_d0f0; pub const DEFAULT_ID_AA64PFR0_EL1: u64 = (0xf << 20) | (0xf << 16) | (1 << 4) | 1; pub const DEFAULT_ID_AA64MMFR0_EL1: u64 = (0xf << 24) | 2; +pub const DEFAULT_ID_AA64ISAR1_EL1: u64 = 1 << 20; pub fn value(register: SystemRegister) -> Option { use SystemRegister::*; Some(match register { @@ -32,7 +33,6 @@ pub fn value(register: SystemRegister) -> Option { CntfrqEl0 => counter_hz, IdAa64dfr0El1 => 6, IdAa64isar0El1 => 2 << 20, - IdAa64isar1El1 => 1 << 20, ClidrEl1 => CLIDR, RevidrEl1 | IdAa64pfr1El1 | IdAa64pfr2El1 | IdAa64zfr0El1 | IdAa64smfr0El1 | IdAa64fpfr0El1 | IdAa64isar3El1 | AidrEl1 | IdAa64dfr1El1 | IdAa64isar2El1 @@ -59,7 +59,7 @@ mod tests { assert_eq!((mmfr0 >> 20) & 15, 0); // FEAT_LSE (Atomics = 2) and FEAT_LRCPC (LDAPR) are implemented. assert_eq!(value(IdAa64isar0El1), Some(2 << 20)); - assert_eq!(value(IdAa64isar1El1), Some(1 << 20)); + assert_eq!(DEFAULT_ID_AA64ISAR1_EL1, 1 << 20); assert_eq!(value(MpidrEl1), Some(1 << 31)); assert_eq!(value(Ttbr0El1), None); } diff --git a/src/emulator/src/aarch64/mod.rs b/src/emulator/src/aarch64/mod.rs index db05489..f6c91c1 100644 --- a/src/emulator/src/aarch64/mod.rs +++ b/src/emulator/src/aarch64/mod.rs @@ -7,6 +7,7 @@ pub mod exception; pub mod fp; pub mod host; pub mod identification; +pub mod pauth; pub mod simd; pub mod simd_struct; pub mod translate; diff --git a/src/emulator/src/aarch64/pauth.rs b/src/emulator/src/aarch64/pauth.rs new file mode 100644 index 0000000..492dc10 --- /dev/null +++ b/src/emulator/src/aarch64/pauth.rs @@ -0,0 +1,865 @@ +//! Pointer authentication (FEAT_PAuth): the QARMA5 `ComputePAC` the architecture +//! defines, the sign, authenticate and strip operations on 48-bit pointers, and the +//! PAuth instructions the JIT hands to the host ([`super::host`]). +//! +//! The emulated CPU advertises the architected QARMA5 algorithm without TBI, EPAC or +//! FPAC (`ID_AA64ISAR1_EL1.APA` = 1). A pointer's PAC then occupies bits 63:56 and 54:48, +//! and authentication failure corrupts bits 62:61 instead of faulting. +//! +//! The cipher follows the pseudocode in the architecture as ported from QEMU's +//! `target/arm/tcg/pauth_helper.c`; the tests check it against that implementation. + +use super::{Cpu, exception}; + +/// A 128-bit PAuth key: `hi` is bits 127:64 (`APxAKeyHi_EL1`), `lo` bits 63:0. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Key { + pub hi: u64, + pub lo: u64, +} + +/// The key the instruction uses, as an index into [`super::cpu::System::pac_keys`] +/// (two words per key, low word first). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum KeyId { + Ia, + Ib, + Da, + Db, + Ga, +} + +impl KeyId { + fn index(self) -> usize { + match self { + Self::Ia => 0, + Self::Ib => 1, + Self::Da => 2, + Self::Db => 3, + Self::Ga => 4, + } + } + + /// The `SCTLR_EL1` bit that enables the key, if any. The generic key has none. + fn enable_bit(self) -> Option { + match self { + Self::Ia => Some(31), + Self::Ib => Some(30), + Self::Da => Some(27), + Self::Db => Some(13), + Self::Ga => None, + } + } + + /// The `keynumber` the architecture reports in an authentication failure. + fn number(self) -> u64 { + match self { + Self::Ia => 0, + Self::Ib => 1, + Self::Da => 2, + Self::Db => 3, + Self::Ga => 4, + } + } +} + +/// The modifier operand: zero, or a register where register 31 is `SP`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Modifier { + Zero, + Reg(u8), +} + +/// One PAuth instruction, decoded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Op { + Sign { + key: KeyId, + reg: u8, + modifier: Modifier, + }, + Auth { + key: KeyId, + reg: u8, + modifier: Modifier, + }, + Strip { + reg: u8, + }, + Pacga { + rd: u8, + rn: u8, + modifier: Modifier, + }, + /// `BRAA`, `BRAB`, `BLRAA`, `BLRAB` and their zero-modifier forms. + Branch { + key: KeyId, + link: bool, + target: u8, + modifier: Modifier, + }, + /// `RETAA` and `RETAB`: return to `LR`, authenticated with `SP`. + Return { + key: KeyId, + }, + /// `ERETAA` and `ERETAB`: return from the exception, authenticating `ELR_EL1`. + ExceptionReturn { + key: KeyId, + }, +} + +/// The hint-space forms (`PACIASP` and friends), which are no-ops without PAuth. +const HINTS: [(u32, Op); 13] = [ + ( + 0xd503_233f, + Op::Sign { + key: KeyId::Ia, + reg: 30, + modifier: Modifier::Reg(31), + }, + ), + ( + 0xd503_23bf, + Op::Auth { + key: KeyId::Ia, + reg: 30, + modifier: Modifier::Reg(31), + }, + ), + ( + 0xd503_237f, + Op::Sign { + key: KeyId::Ib, + reg: 30, + modifier: Modifier::Reg(31), + }, + ), + ( + 0xd503_23ff, + Op::Auth { + key: KeyId::Ib, + reg: 30, + modifier: Modifier::Reg(31), + }, + ), + ( + 0xd503_211f, + Op::Sign { + key: KeyId::Ia, + reg: 17, + modifier: Modifier::Reg(16), + }, + ), + ( + 0xd503_219f, + Op::Auth { + key: KeyId::Ia, + reg: 17, + modifier: Modifier::Reg(16), + }, + ), + ( + 0xd503_215f, + Op::Sign { + key: KeyId::Ib, + reg: 17, + modifier: Modifier::Reg(16), + }, + ), + ( + 0xd503_21df, + Op::Auth { + key: KeyId::Ib, + reg: 17, + modifier: Modifier::Reg(16), + }, + ), + ( + 0xd503_231f, + Op::Sign { + key: KeyId::Ia, + reg: 30, + modifier: Modifier::Zero, + }, + ), + ( + 0xd503_239f, + Op::Auth { + key: KeyId::Ia, + reg: 30, + modifier: Modifier::Zero, + }, + ), + ( + 0xd503_235f, + Op::Sign { + key: KeyId::Ib, + reg: 30, + modifier: Modifier::Zero, + }, + ), + ( + 0xd503_23df, + Op::Auth { + key: KeyId::Ib, + reg: 30, + modifier: Modifier::Zero, + }, + ), + (0xd503_20ff, Op::Strip { reg: 30 }), +]; + +fn decode_word(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = rd; + // Data-processing, one source: PAC and AUT (opcodes 0-15), and XPAC (16-17). + if word & 0xffff_0000 == 0xdac1_0000 { + let opcode = (word >> 10) & 0x3f; + let key = [KeyId::Ia, KeyId::Ib, KeyId::Da, KeyId::Db][(opcode & 3) as usize]; + let modifier = if opcode & 8 != 0 { + Modifier::Zero + } else { + Modifier::Reg(rn) + }; + return match opcode { + 0..=15 if opcode & 4 == 0 => Some(Op::Sign { + key, + reg: rd, + modifier, + }), + 0..=15 => Some(Op::Auth { + key, + reg: rd, + modifier, + }), + 16 | 17 if rn == 31 => Some(Op::Strip { reg: rd }), + _ => None, + }; + } + // Data-processing, two sources: PACGA Xd, Xn, Xm|SP. + if word & 0xffe0_fc00 == 0x9ac0_3000 { + let rm = ((word >> 16) & 31) as u8; + return Some(Op::Pacga { + rd, + rn, + modifier: Modifier::Reg(rm), + }); + } + // Branches with authentication. Bit 10 selects the B key; bit 11 is always set. + let b_key = if word & 0x400 != 0 { + KeyId::Ib + } else { + KeyId::Ia + }; + if word & 0xffff_f800 == 0xd71f_0800 { + return Some(Op::Branch { + key: b_key, + link: false, + target: rn, + modifier: Modifier::Reg(rm), + }); + } + if word & 0xffff_f81f == 0xd61f_081f { + return Some(Op::Branch { + key: b_key, + link: false, + target: rn, + modifier: Modifier::Zero, + }); + } + if word & 0xffff_f800 == 0xd73f_0800 { + return Some(Op::Branch { + key: b_key, + link: true, + target: rn, + modifier: Modifier::Reg(rm), + }); + } + if word & 0xffff_f81f == 0xd63f_081f { + return Some(Op::Branch { + key: b_key, + link: true, + target: rn, + modifier: Modifier::Zero, + }); + } + if word & 0xffff_fbff == 0xd65f_0bff { + return Some(Op::Return { key: b_key }); + } + if word & 0xffff_fbff == 0xd69f_0bff { + return Some(Op::ExceptionReturn { key: b_key }); + } + HINTS + .iter() + .find(|(hint, _)| *hint == word) + .map(|&(_, op)| op) +} + +/// Whether [`run`] implements `word`. +pub fn is_supported(word: u32) -> bool { + decode_word(word).is_some() +} + +/// The 48-bit pointer's bits 63:48 all equal bit 47, so the pointer is canonical. +fn is_canonical(ptr: u64) -> bool { + let top = ptr >> 48; + top == 0 || top == 0xffff +} + +/// Bit 55 selects the address range. A canonical pointer's bits 63:48 equal it. +fn original_ptr(ptr: u64) -> u64 { + if ptr & (1 << 55) != 0 { + ptr | 0xffff_0000_0000_0000 + } else { + ptr & 0x0000_ffff_ffff_ffff + } +} + +/// The PAC of a pointer: the `ComputePAC` of its extension bits filled in. +pub fn add_pac(ptr: u64, modifier: u64, key: Key) -> u64 { + let ext = (ptr >> 63) & 1; + let ext_ptr = (ptr & 0x0000_ffff_ffff_ffff) + | if (ptr >> 63) & 1 != 0 { + 0xffff_0000_0000_0000 + } else { + 0 + }; + let mut pac = compute_pac(ext_ptr, modifier, key); + if !is_canonical(ptr) { + pac ^= 1 << 62; + } + let pac = pac & !((1 << 55) | 0x0000_ffff_ffff_ffff); + pac | (ext << 55) | (ptr & 0x0000_ffff_ffff_ffff) +} + +/// Authenticate `ptr` against `modifier`. On success the pointer is returned with its +/// PAC removed. On failure its bits 62:61 carry an error code and it is non-canonical, +/// so its next use faults. +pub fn auth(ptr: u64, modifier: u64, key: Key, key_number: u64) -> u64 { + let orig = original_ptr(ptr); + let pac = compute_pac(orig, modifier, key); + let cmp_mask = 0xff7f_0000_0000_0000u64; + if (pac ^ ptr) & cmp_mask != 0 { + let error = (key_number << 1) | (key_number ^ 1); + return (orig & !(3 << 61)) | ((error & 3) << 61); + } + orig +} + +/// Remove the PAC from a pointer. +pub fn strip(ptr: u64) -> u64 { + original_ptr(ptr) +} + +/// `PACGA`: a 32-bit generic MAC in the top half of the result. +pub fn pacga(data: u64, modifier: u64, key: Key) -> u64 { + compute_pac(data, modifier, key) & 0xffff_ffff_0000_0000 +} + +const RC: [u64; 5] = [ + 0x0000_0000_0000_0000, + 0x1319_8A2E_0370_7344, + 0xA409_3822_299F_31D0, + 0x082E_FA98_EC4E_6C89, + 0x4528_21E6_38D0_1377, +]; +const ALPHA: u64 = 0xC0AC_29B7_C97C_50DD; +const SBOX: [u8; 16] = [ + 0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe, 0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa, +]; +const INV_SBOX: [u8; 16] = [ + 0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9, 0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3, +]; + +/// Cell permutations as `(source nibble, destination nibble)`. +const CELL_SHUFFLE: [(u8, u8); 16] = [ + (13, 0), + (6, 1), + (11, 2), + (0, 3), + (7, 4), + (12, 5), + (1, 6), + (10, 7), + (8, 8), + (3, 9), + (14, 10), + (5, 11), + (2, 12), + (9, 13), + (4, 14), + (15, 15), +]; +const CELL_INV_SHUFFLE: [(u8, u8); 16] = [ + (3, 0), + (6, 1), + (12, 2), + (9, 3), + (14, 4), + (11, 5), + (1, 6), + (4, 7), + (8, 8), + (13, 9), + (7, 10), + (2, 11), + (5, 12), + (0, 13), + (10, 14), + (15, 15), +]; +/// The tweak permutations, with the cells the tweak LFSR steps marked. +const TWEAK_SHUFFLE: [(u8, u8, bool); 16] = [ + (4, 0, false), + (5, 1, false), + (6, 2, true), + (7, 3, false), + (11, 4, true), + (2, 5, false), + (3, 6, false), + (8, 7, true), + (12, 8, false), + (13, 9, false), + (14, 10, false), + (15, 11, true), + (0, 12, true), + (1, 13, false), + (10, 14, true), + (9, 15, true), +]; +const TWEAK_INV_SHUFFLE: [(u8, u8, bool); 16] = [ + (12, 0, true), + (13, 1, false), + (5, 2, false), + (6, 3, false), + (0, 4, false), + (1, 5, false), + (2, 6, true), + (3, 7, false), + (7, 8, true), + (15, 9, true), + (14, 10, true), + (4, 11, true), + (8, 12, false), + (9, 13, false), + (10, 14, false), + (11, 15, true), +]; + +fn nibble(value: u64, n: u8) -> u64 { + (value >> (4 * n)) & 0xf +} + +fn permute(value: u64, table: &[(u8, u8)]) -> u64 { + table + .iter() + .fold(0, |out, &(src, dst)| out | nibble(value, src) << (4 * dst)) +} + +fn permute_tweak(value: u64, table: &[(u8, u8, bool)], step: fn(u64) -> u64) -> u64 { + table.iter().fold(0, |out, &(src, dst, stepped)| { + let cell = nibble(value, src); + let cell = if stepped { step(cell) } else { cell }; + out | cell << (4 * dst) + }) +} + +fn cell_shuffle(value: u64) -> u64 { + permute(value, &CELL_SHUFFLE) +} + +fn cell_inv_shuffle(value: u64) -> u64 { + permute(value, &CELL_INV_SHUFFLE) +} + +fn tweak_cell_rot(cell: u64) -> u64 { + (cell >> 1) | (((cell ^ (cell >> 1)) & 1) << 3) +} + +fn tweak_cell_inv_rot(cell: u64) -> u64 { + ((cell << 1) & 0xf) | ((cell & 1) ^ (cell >> 3)) +} + +fn tweak_shuffle(value: u64) -> u64 { + permute_tweak(value, &TWEAK_SHUFFLE, tweak_cell_rot) +} + +fn tweak_inv_shuffle(value: u64) -> u64 { + permute_tweak(value, &TWEAK_INV_SHUFFLE, tweak_cell_inv_rot) +} + +fn rot_cell(cell: u64, n: u32) -> u64 { + let doubled = cell | cell << 4; + (doubled >> (4 - n)) & 0xf +} + +/// The MixColumns-like diffusion layer, applied to each column of the 4x4 cell matrix. +fn mult(value: u64) -> u64 { + let mut out = 0; + for column in 0..4u8 { + let i0 = nibble(value, column); + let i4 = nibble(value, column + 4); + let i8 = nibble(value, column + 8); + let ic = nibble(value, column + 12); + let t0 = rot_cell(i8, 1) ^ rot_cell(i4, 2) ^ rot_cell(i0, 1); + let t1 = rot_cell(ic, 1) ^ rot_cell(i4, 1) ^ rot_cell(i0, 2); + let t2 = rot_cell(ic, 2) ^ rot_cell(i8, 1) ^ rot_cell(i0, 1); + let t3 = rot_cell(ic, 1) ^ rot_cell(i8, 2) ^ rot_cell(i4, 1); + out |= t3 << (4 * column); + out |= t2 << (4 * (column + 4)); + out |= t1 << (4 * (column + 8)); + out |= t0 << (4 * (column + 12)); + } + out +} + +fn sub(value: u64, table: &[u8; 16]) -> u64 { + (0..16).fold(0, |out, n| { + out | u64::from(table[nibble(value, n) as usize]) << (4 * n) + }) +} + +/// `ComputePAC` of the architecture: the QARMA5 cipher over `data`, keyed by `key`, +/// tweaked by `modifier`. +pub fn compute_pac(data: u64, modifier: u64, key: Key) -> u64 { + let key0 = key.hi; + let key1 = key.lo; + let modk0 = (key0 << 63) | ((key0 >> 1) ^ (key0 >> 63)); + let mut runningmod = modifier; + let mut working = data ^ key0; + for (i, rc) in RC.iter().enumerate() { + working ^= key1 ^ runningmod; + working ^= rc; + if i > 0 { + working = mult(cell_shuffle(working)); + } + working = sub(working, &SBOX); + runningmod = tweak_shuffle(runningmod); + } + working ^= modk0 ^ runningmod; + working = mult(cell_shuffle(working)); + working = sub(working, &SBOX); + working = mult(cell_shuffle(working)); + working ^= key1; + working = sub(cell_inv_shuffle(working), &INV_SBOX); + working = cell_inv_shuffle(mult(working)); + working ^= key0 ^ runningmod; + for i in 0..=4usize { + working = sub(working, &INV_SBOX); + if i < 4 { + working = cell_inv_shuffle(mult(working)); + } + runningmod = tweak_inv_shuffle(runningmod); + working ^= RC[4 - i] ^ (key1 ^ runningmod) ^ ALPHA; + } + working ^ modk0 +} + +fn key(cpu: &Cpu, id: KeyId) -> Key { + let keys = &cpu.system.pac_keys; + Key { + lo: keys[2 * id.index()], + hi: keys[2 * id.index() + 1], + } +} + +/// Whether the key is enabled for the current translation regime (`SCTLR_EL1`). +fn enabled(cpu: &Cpu, id: KeyId) -> bool { + id.enable_bit() + .is_none_or(|bit| cpu.system.sctlr_el1 & (1 << bit) != 0) +} + +/// Register `n` read as `Xn|SP` (31 is the stack pointer). +fn read_sp(cpu: &Cpu, n: u8) -> u64 { + if n == 31 { cpu.sp } else { cpu.x[n as usize] } +} + +/// Register `n` read as `Xn` (31 is zero). +fn read_zr(cpu: &Cpu, n: u8) -> u64 { + if n == 31 { 0 } else { cpu.x[n as usize] } +} + +fn write_x(cpu: &mut Cpu, n: u8, value: u64) { + if n != 31 { + cpu.x[n as usize] = value; + } +} + +fn modifier_value(cpu: &Cpu, modifier: Modifier) -> u64 { + match modifier { + Modifier::Zero => 0, + Modifier::Reg(n) => read_sp(cpu, n), + } +} + +/// Execute the PAuth instruction `word` on `cpu`. `word` must satisfy [`is_supported`]. +/// `cpu.pc` is the address of the next instruction, which a link writes to `LR`. +pub fn run(cpu: &mut Cpu, word: u32) { + let op = decode_word(word).expect("pauth::run is only called for supported words"); + match op { + Op::Sign { + key: id, + reg, + modifier, + } => { + if enabled(cpu, id) { + let value = add_pac( + cpu.x[reg as usize], + modifier_value(cpu, modifier), + key(cpu, id), + ); + write_x(cpu, reg, value); + } + } + Op::Auth { + key: id, + reg, + modifier, + } => { + if enabled(cpu, id) { + let value = auth( + cpu.x[reg as usize], + modifier_value(cpu, modifier), + key(cpu, id), + id.number(), + ); + write_x(cpu, reg, value); + } + } + Op::Strip { reg } => { + let value = strip(cpu.x[reg as usize]); + write_x(cpu, reg, value); + } + Op::Pacga { rd, rn, modifier } => { + let value = pacga( + read_zr(cpu, rn), + modifier_value(cpu, modifier), + key(cpu, KeyId::Ga), + ); + write_x(cpu, rd, value); + } + Op::Branch { + key: id, + link, + target, + modifier, + } => { + let next = cpu.pc; + let mut destination = read_zr(cpu, target); + if enabled(cpu, id) { + destination = auth( + destination, + modifier_value(cpu, modifier), + key(cpu, id), + id.number(), + ); + } + if link { + write_x(cpu, 30, next); + } + cpu.pc = destination; + } + Op::Return { key: id } => { + let mut destination = cpu.x[30]; + if enabled(cpu, id) { + destination = auth(destination, cpu.sp, key(cpu, id), id.number()); + } + cpu.pc = destination; + } + Op::ExceptionReturn { key: id } => { + let mut elr = cpu.system.elr_el1; + if enabled(cpu, id) { + elr = auth(elr, cpu.sp, key(cpu, id), id.number()); + } + cpu.system.elr_el1 = elr; + cpu.pc = exception::eret(cpu); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The key the QEMU reference was run with. + const KEY: Key = Key { + hi: 0xec28_02d4_e0a4_88e9, + lo: 0x84be_85ce_9804_e94b, + }; + + /// `ComputePAC` values from QEMU's `pauth_computepac_architected`, compiled from its + /// source with the same key: `(data, modifier, pac)`. + const REFERENCE: [(u64, u64, u64); 6] = [ + ( + 0x0000_0000_0000_0000, + 0x0000_0000_0000_0000, + 0xbc2f_6e1c_6e1a_f447, + ), + ( + 0xffff_fe00_0700_4000, + 0x477d_469d_ec0b_8762, + 0xc1b6_9701_e911_0f0c, + ), + ( + 0x0000_0008_1234_5678, + 0x0000_0000_0000_0000, + 0xc3ad_cffe_19ff_629c, + ), + ( + 0x0000_0008_1234_5678, + 0x2222_2222_2222_2222, + 0xfe68_3d52_004b_603d, + ), + ( + 0x1234_5678_1234_5678, + 0x0000_0000_0000_0000, + 0x29e3_4d27_b244_88a6, + ), + ( + 0x1234_5678_1234_5678, + 0x477d_469d_ec0b_8762, + 0x2b7a_744b_38ed_59f6, + ), + ]; + + #[test] + fn compute_pac_matches_the_architected_reference() { + for (data, modifier, pac) in REFERENCE { + assert_eq!(compute_pac(data, modifier, KEY), pac, "data {data:#x}"); + } + } + + #[test] + fn sign_then_authenticate_returns_the_pointer_and_strip_removes_the_pac() { + let ptr = 0x0000_0008_1234_5678; + let signed = add_pac(ptr, 0x2222, KEY); + assert_ne!(signed, ptr, "a PAC is inserted"); + assert_eq!(auth(signed, 0x2222, KEY, 0), ptr); + assert_eq!(strip(signed), ptr); + } + + #[test] + fn a_kernel_pointer_round_trips_through_sign_and_auth() { + let ptr = 0xffff_8000_8004_0000; + let signed = add_pac(ptr, 0x1234, KEY); + assert_eq!(auth(signed, 0x1234, KEY, 2), ptr); + } + + #[test] + fn a_wrong_modifier_or_key_leaves_a_non_canonical_pointer() { + let ptr = 0x0000_0008_1234_5678; + let signed = add_pac(ptr, 0x2222, KEY); + let other_key = Key { + lo: KEY.lo ^ 1, + ..KEY + }; + for failed in [ + auth(signed, 0x3333, KEY, 0), + auth(signed, 0x2222, other_key, 0), + ] { + assert!(!is_canonical(failed), "{failed:#x} must not be usable"); + assert_eq!((failed >> 61) & 3, 1, "error code for key A"); + } + } + + #[test] + fn a_non_canonical_input_is_not_signed_into_a_valid_pointer() { + // A pointer whose extension bits are wrong gets a corrupted PAC, so it cannot + // authenticate later. + let bad = 0x1234_0008_1234_5678; + let signed = add_pac(bad, 0, KEY); + assert!(!is_canonical(auth(signed, 0, KEY, 0))); + } + + #[test] + fn pacga_keeps_only_the_top_word() { + let mac = pacga(0x1234, 0x5678, KEY); + assert_eq!(mac & 0xffff_ffff, 0); + assert_eq!( + mac, + compute_pac(0x1234, 0x5678, KEY) & 0xffff_ffff_0000_0000 + ); + } + + #[test] + fn the_instruction_encodings_decode_as_their_names() { + // Assembled with GNU as for armv8.3-a. + assert_eq!( + decode_word(0xdac1_0020), + Some(Op::Sign { + key: KeyId::Ia, + reg: 0, + modifier: Modifier::Reg(1) + }) + ); + assert_eq!( + decode_word(0xdac1_11ac), + Some(Op::Auth { + key: KeyId::Ia, + reg: 12, + modifier: Modifier::Reg(13) + }) + ); + assert_eq!( + decode_word(0xdac1_23f0), + Some(Op::Sign { + key: KeyId::Ia, + reg: 16, + modifier: Modifier::Zero + }) + ); + assert_eq!(decode_word(0xdac1_43f8), Some(Op::Strip { reg: 24 })); + assert_eq!( + decode_word(0xd71f_0c43), + Some(Op::Branch { + key: KeyId::Ib, + link: false, + target: 2, + modifier: Modifier::Reg(3) + }) + ); + assert_eq!( + decode_word(0xd63f_095f), + Some(Op::Branch { + key: KeyId::Ia, + link: true, + target: 10, + modifier: Modifier::Zero + }) + ); + assert_eq!( + decode_word(0xd65f_0fff), + Some(Op::Return { key: KeyId::Ib }) + ); + assert_eq!( + decode_word(0xd503_233f), + Some(Op::Sign { + key: KeyId::Ia, + reg: 30, + modifier: Modifier::Reg(31) + }) + ); + // Plain BR and ordinary one-source data processing are not PAuth. + assert!(!is_supported(0xd61f_0000)); + assert!(!is_supported(0xdac0_0000)); + } + + #[test] + fn a_pac_instruction_signs_its_register_and_a_branch_jumps_to_the_authenticated_target() { + let mut cpu = Cpu::default(); + cpu.system.sctlr_el1 = 1 << 31; + cpu.system.pac_keys[0] = KEY.lo; + cpu.system.pac_keys[1] = KEY.hi; + let ptr = 0x0000_0008_1234_5678; + cpu.x[0] = ptr; + cpu.x[1] = 0x2222; + run(&mut cpu, 0xdac1_0020); + assert_eq!(cpu.x[0], add_pac(ptr, 0x2222, KEY)); + cpu.x[2] = add_pac(0x4000, 0x2222, KEY); + cpu.x[3] = 0x2222; + cpu.pc = 0x1000; + run(&mut cpu, 0xd71f_0800 | 2 << 5 | 3); + assert_eq!(cpu.pc, 0x4000); + } +} diff --git a/src/emulator/src/system/m1.rs b/src/emulator/src/system/m1.rs index b49cdec..2b7266b 100644 --- a/src/emulator/src/system/m1.rs +++ b/src/emulator/src/system/m1.rs @@ -42,6 +42,9 @@ pub const ID_AA64PFR0_EL1: u64 = 0x11; /// `ID_AA64MMFR0_EL1`: 4 KiB and 16 KiB translation granules (the Asahi kernel is built /// for 16 KiB pages and refuses a CPU without one), no 64 KiB granule, 40-bit PA. pub const ID_AA64MMFR0_EL1: u64 = (0xf << 24) | (1 << 20) | 2; +/// `ID_AA64ISAR1_EL1`: LRCPC, and the architected QARMA5 for address (APA) and generic +/// (GPA) pointer authentication. No implementation-defined algorithm (API, GPI). +pub const ID_AA64ISAR1_EL1: u64 = (1 << 20) | (1 << 24) | (1 << 4); /// The machine runs one vCPU. The device tree lists eight cores, but their /// `cpu-release-addr` is zero, so Linux's spin-table code does not start them. @@ -52,6 +55,7 @@ pub fn identify(system: &mut crate::aarch64::cpu::System) { system.midr_el1 = MIDR_EL1; system.id_aa64pfr0_el1 = ID_AA64PFR0_EL1; system.id_aa64mmfr0_el1 = ID_AA64MMFR0_EL1; + system.id_aa64isar1_el1 = ID_AA64ISAR1_EL1; } /// The SoC devices on the bus: the AIC, the power-state blocks, the console, and the From 1f2656d437b1dc37dac6272688412337c2cfa70a Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Wed, 7 Oct 2026 20:37:09 -0700 Subject: [PATCH 04/13] feat(emulator): add EL2 regime, VHE, and SPTM monitor loader - EL2 banks, exception entry/ERET to EL2, EL2 translation regime (TTBR0/TTBR1 with TCR_EL2 layouts), HCR_EL2.TGE routing, and HCR_EL2.E2H (VHE) redirection of EL1 register names from EL2. - Stored IMP-DEF bank for S3_{4,5,6}_c15 registers the SPTM monitor configures. - system::monitor: place Apple's sptm.t8103 image in DRAM, boot structure with the linear-map fields, ADT placeholder-flag cleaning, and chosen/memory-map region insertion. - Debug hooks: Machine::set_step_trace (one instruction per block, reports changed system registers including exception entries) and Machine::set_memory_trace (translated loads and stores). - ID_AA64PFR0_EL1 advertises EL2. --- README.md | 30 +- src/emulator/src/aarch64/cache.rs | 18 +- src/emulator/src/aarch64/compile.rs | 89 ++++ src/emulator/src/aarch64/cpu.rs | 46 +++ src/emulator/src/aarch64/decode.rs | 85 +++- src/emulator/src/aarch64/exception.rs | 142 +++++-- src/emulator/src/aarch64/gxf.rs | 74 ++++ src/emulator/src/aarch64/host.rs | 9 +- src/emulator/src/aarch64/mod.rs | 1 + src/emulator/src/aarch64/translate.rs | 114 +++++- src/emulator/src/system/m1.rs | 13 +- src/emulator/src/system/machine.rs | 257 +++++++++++- src/emulator/src/system/mod.rs | 9 +- src/emulator/src/system/monitor.rs | 558 ++++++++++++++++++++++++++ src/emulator/src/system/xnu.rs | 6 + 15 files changed, 1384 insertions(+), 67 deletions(-) create mode 100644 src/emulator/src/aarch64/gxf.rs create mode 100644 src/emulator/src/system/monitor.rs diff --git a/README.md b/README.md index 68bbd32..6a9d0e8 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,34 @@ The Linux kernel boots through the AIC and the S5L console, probes its power dom and panics on the missing root filesystem (`VFS: Unable to mount root fs`), which is the expected result without one. `--initrd`, `--disk` and `--smp` above 1 are rejected on this board. A kernelcache from the restore image (`MH_FILESET`) loads, with its ADT -as `--dtb`, and runs its first blocks; it does not get past the early boot CPU -table yet, and it needs PAuth and chained fixups before it can go further. +as `--dtb`, and enters at its reset trampoline with the cold-boot reset type. It runs +several hundred blocks of early boot, then executes `genter` (guarded execution) before +it has set `VBAR_EL1`. The kernel expects iBoot to have enabled GXF and set its monitor +entry, which volt does not provide; chained fixups are still not applied. + +The EL2 regime is modelled: `CurrentEl` reports EL2, exceptions and `ERET` target +`VBAR_EL2`/`ELR_EL2`/`SPSR_EL2`, translation walks `TTBR0_EL2`/`TCR_EL2`, and +`HCR_EL2.E2H` (VHE) redirects EL1 register names accessed from EL2 to their EL2 banks. +`system::monitor` loads Apple's SPTM monitor (`sptm.t8103.release`) the way the boot +firmware does: the image placed in DRAM at its physical address, the MMU off, EL2, and +`x0` a boot structure (DRAM's virtual and physical base and size, the device tree, a +scratch buffer). The monitor builds its own translation tables, turns the MMU on, and +reads `chosen/memory-map` for the image regions iBoot would have recorded. Its strings +(`sptm_fixup`, `unsupported auth_rebase key`) and a loop over the kernel's range suggest +it applies the kernelcache's chained fixups itself; that is inferred, not yet confirmed. +It runs about four million instructions on the real kernelcache before it panics with +`init_get_image_region: error -1 looking up image region 'TXM-ro'`, written to its scratch +buffer. The map holds one entry per image (`SPTM`, `TXM`, `BootKC`, `AuxKC`, `CL4`) and +part (`-ro`, `-rx`, `-rw`, `-bx`, `-le`, `-rs`, plus `-entry`, `-virt`, `-exception`); +volt supplies `BootKC-rs` and `BootKC-ro` only, and does not load TXM. GXF (`genter`, +`gexit`, `GXF_*`, `VBAR_GL1`) and SPRR are not modelled; `genter` raises an undefined +instruction. + +Two debugging hooks make this tractable. `Machine::set_step_trace` runs one instruction +per block and reports each step with the system registers it changed, exception entries +included. `Machine::set_memory_trace` reports every translated load and store (atomics +and SIMD structure accesses excepted). Installing either turns the inline data TLB off, +so a traced run is much slower than an untraced one. Not modelled: any arm-io device other than the AIC, the UART and the power-state blocks. Those addresses read as zero and writes are dropped, and the boot report counts diff --git a/src/emulator/src/aarch64/cache.rs b/src/emulator/src/aarch64/cache.rs index 350fbc7..604a25a 100644 --- a/src/emulator/src/aarch64/cache.rs +++ b/src/emulator/src/aarch64/cache.rs @@ -26,6 +26,8 @@ pub struct Cache { /// Where blocks come from on a local miss. Private to this cache unless /// built with `with_shared`. shared: Arc, + /// Instructions one block scan may take: `MAX_INSNS`, or 1 when single-stepping. + max_insns: usize, } impl Default for Cache { fn default() -> Self { @@ -34,6 +36,7 @@ impl Default for Cache { index: HashMap::new(), direct: alloc_direct(), shared: Arc::new(SharedBlocks::new(false)), + max_insns: MAX_INSNS, } } } @@ -192,6 +195,17 @@ impl Cache { pub fn new() -> Self { Self::default() } + + /// A cache whose blocks are one instruction each, so a trace sees every step. + pub fn single_step(inline_memory: bool) -> Self { + let mut cache = if inline_memory { + Self::with_inline_memory() + } else { + Self::new() + }; + cache.max_insns = 1; + cache + } /// Whether blocks from this cache read `Cpu::dtlb`. pub fn inline_memory(&self) -> bool { self.shared.inline_memory() @@ -326,7 +340,7 @@ impl Cache { let mut count = 0usize; let mut complete = false; let mut last = None; - for n in 0..MAX_INSNS { + for n in 0..self.max_insns { let at = cpu .pc .checked_add(count as u64) @@ -366,7 +380,7 @@ impl Cache { // Never straddle a page: a block inside one page is validated by // that page's version alone, and the next page is translated and // checked when its own block starts. - complete = n + 1 == MAX_INSNS || (pc + count as u64) & 0xfff == 0; + complete = n + 1 == self.max_insns || (pc + count as u64) & 0xfff == 0; if complete { break; } diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index 0858b93..75691bc 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -2446,6 +2446,19 @@ fn system_offset(r: SystemRegister) -> Option { CntpCtlEl0 => offset_of!(CpuSystem, cntp_ctl_el0), ActlrEl1 => offset_of!(CpuSystem, actlr_el1), VbarEl1 => offset_of!(CpuSystem, vbar_el1), + SctlrEl2 => offset_of!(CpuSystem, sctlr_el2), + TcrEl2 => offset_of!(CpuSystem, tcr_el2), + Ttbr0El2 => offset_of!(CpuSystem, ttbr0_el2), + Ttbr1El2 => offset_of!(CpuSystem, ttbr1_el2), + MairEl2 => offset_of!(CpuSystem, mair_el2), + VbarEl2 => offset_of!(CpuSystem, vbar_el2), + ElrEl2 => offset_of!(CpuSystem, elr_el2), + SpsrEl2 => offset_of!(CpuSystem, spsr_el2), + EsrEl2 => offset_of!(CpuSystem, esr_el2), + FarEl2 => offset_of!(CpuSystem, far_el2), + HcrEl2 => offset_of!(CpuSystem, hcr_el2), + TpidrEl2 => offset_of!(CpuSystem, tpidr_el2), + CntvoffEl2 => offset_of!(CpuSystem, cntvoff_el2), CpacrEl1 => offset_of!(CpuSystem, cpacr_el1), SctlrEl1 => offset_of!(CpuSystem, sctlr_el1), Ttbr0El1 => offset_of!(CpuSystem, ttbr0_el1), @@ -2468,6 +2481,10 @@ fn system_offset(r: SystemRegister) -> Option { Fpsr => offset_of!(CpuSystem, fpsr), Daif => offset_of!(CpuSystem, daif), Apple(slot) => offset_of!(CpuSystem, apple) + 8 * usize::from(slot), + // After `system` in `Cpu`: relative to it, like every other register here. + AppleBank(slot) => { + offset_of!(Cpu, apple_bank) - offset_of!(Cpu, system) + 8 * usize::from(slot) + } MidrEl1 => offset_of!(CpuSystem, midr_el1), IdAa64pfr0El1 => offset_of!(CpuSystem, id_aa64pfr0_el1), IdAa64mmfr0El1 => offset_of!(CpuSystem, id_aa64mmfr0_el1), @@ -2476,6 +2493,55 @@ fn system_offset(r: SystemRegister) -> Option { _ => return None, }) } + +/// EL1 registers that `HCR_EL2.E2H` redirects to their EL2 banks when accessed from EL2: +/// (EL1 field offset, EL2 field offset). +fn vhe_banks(r: SystemRegister) -> Option<(usize, usize)> { + use SystemRegister::*; + Some(match r { + SctlrEl1 => ( + offset_of!(CpuSystem, sctlr_el1), + offset_of!(CpuSystem, sctlr_el2), + ), + Ttbr0El1 => ( + offset_of!(CpuSystem, ttbr0_el1), + offset_of!(CpuSystem, ttbr0_el2), + ), + Ttbr1El1 => ( + offset_of!(CpuSystem, ttbr1_el1), + offset_of!(CpuSystem, ttbr1_el2), + ), + TcrEl1 => ( + offset_of!(CpuSystem, tcr_el1), + offset_of!(CpuSystem, tcr_el2), + ), + MairEl1 => ( + offset_of!(CpuSystem, mair_el1), + offset_of!(CpuSystem, mair_el2), + ), + VbarEl1 => ( + offset_of!(CpuSystem, vbar_el1), + offset_of!(CpuSystem, vbar_el2), + ), + ElrEl1 => ( + offset_of!(CpuSystem, elr_el1), + offset_of!(CpuSystem, elr_el2), + ), + SpsrEl1 => ( + offset_of!(CpuSystem, spsr_el1), + offset_of!(CpuSystem, spsr_el2), + ), + EsrEl1 => ( + offset_of!(CpuSystem, esr_el1), + offset_of!(CpuSystem, esr_el2), + ), + FarEl1 => ( + offset_of!(CpuSystem, far_el1), + offset_of!(CpuSystem, far_el2), + ), + _ => return None, + }) +} fn lower_system(l: &Lower, a: System) { use SystemRegister::*; l.flush_counter(); @@ -2518,6 +2584,29 @@ fn lower_system(l: &Lower, a: System) { } return; } + if let Some((el1, el2)) = vhe_banks(a.register) { + // HCR_EL2.E2H: EL1 register names accessed from EL2 reach the EL2 banks. + let current = l.cv(l.u64, l.load(l.u8, at + offset_of!(CpuSystem, el))); + let el2_mode = l.cmp(C::Eq, current, l.k(l.u64, 2)); + let e2h = l.cmp( + C::Ne, + l.imm( + l.u64, + B::BitAnd, + l.load(l.u64, at + offset_of!(CpuSystem, hcr_el2)), + super::cpu::HCR_E2H, + ), + l.k(l.u64, 0), + ); + let redirect = l.bin(l.boolean, B::BitAnd, el2_mode, e2h); + let p = l.sel(l.ptr, redirect, l.addr(at + el2), l.addr(at + el1)); + if a.read { + l.put(a.rt, l.load_ptr(l.u64, p)); + } else { + l.store_ptr(p, l.reg(l.u64, a.rt, true)); + } + return; + } match a.register { CurrentEl => l.put( a.rt, diff --git a/src/emulator/src/aarch64/cpu.rs b/src/emulator/src/aarch64/cpu.rs index 802f7b6..01af196 100644 --- a/src/emulator/src/aarch64/cpu.rs +++ b/src/emulator/src/aarch64/cpu.rs @@ -153,8 +153,27 @@ pub struct Cpu { pub v: [[u64; 2]; 32], /// Inline data TLB for generated loads and stores. Filled by the machine. pub dtlb: Dtlb, + /// Apple IMP-DEF registers of the `op1` 0 to 6 banks the SPTM monitor configures. It + /// is the last field so growing it never moves the fields generated code reaches + /// with short immediate offsets (`v`, `dtlb`). + pub apple_bank: ImpDefBank, } +/// `S3__c15` registers, indexed by `decode::apple_bank_slot`. +#[derive(Clone, Debug)] +pub struct ImpDefBank(pub [u64; crate::aarch64::decode::APPLE_BANK_SLOTS]); + +impl Default for ImpDefBank { + fn default() -> Self { + Self([0; crate::aarch64::decode::APPLE_BANK_SLOTS]) + } +} + +/// `HCR_EL2.TGE` (bit 27): exceptions taken from EL0 route to EL2. +pub const HCR_TGE: u64 = 1 << 27; +/// `HCR_EL2.E2H` (bit 34): VHE. EL1 register names accessed from EL2 reach the EL2 banks. +pub const HCR_E2H: u64 = 1 << 34; + #[derive(Clone, Debug)] #[repr(C)] pub struct System { @@ -199,6 +218,20 @@ pub struct System { pub icc_igrpen0_el1: u64, pub cntp_ctl_el0: u64, pub actlr_el1: u64, + /// EL2 banks (SPTM runs at EL2, GL2). `SP_EL2` is `sp_el[2]`. + pub sctlr_el2: u64, + pub tcr_el2: u64, + pub ttbr0_el2: u64, + pub ttbr1_el2: u64, + pub mair_el2: u64, + pub vbar_el2: u64, + pub elr_el2: u64, + pub spsr_el2: u64, + pub esr_el2: u64, + pub far_el2: u64, + pub hcr_el2: u64, + pub tpidr_el2: u64, + pub cntvoff_el2: u64, /// Apple IMP-DEF configuration registers, indexed by `decode::SystemRegister::Apple`. pub apple: [u64; crate::aarch64::decode::APPLE_SLOTS], /// `MIDR_EL1`, `ID_AA64PFR0_EL1` and `ID_AA64MMFR0_EL1`: the identity this CPU @@ -228,6 +261,19 @@ impl Default for System { ttbr0_el1: 0, ttbr1_el1: 0, tcr_el1: 0, + sctlr_el2: 0, + tcr_el2: 0, + ttbr0_el2: 0, + ttbr1_el2: 0, + mair_el2: 0, + vbar_el2: 0, + elr_el2: 0, + spsr_el2: 0, + esr_el2: 0, + far_el2: 0, + hcr_el2: 0, + tpidr_el2: 0, + cntvoff_el2: 0, mair_el1: 0, tpidr_el0: 0, tpidrro_el0: 0, diff --git a/src/emulator/src/aarch64/decode.rs b/src/emulator/src/aarch64/decode.rs index 530787f..e733cef 100644 --- a/src/emulator/src/aarch64/decode.rs +++ b/src/emulator/src/aarch64/decode.rs @@ -520,6 +520,19 @@ pub enum SystemRegister { SpEl0, SpEl1, SpEl2, + SctlrEl2, + TcrEl2, + Ttbr0El2, + Ttbr1El2, + MairEl2, + VbarEl2, + ElrEl2, + SpsrEl2, + EsrEl2, + FarEl2, + HcrEl2, + TpidrEl2, + CntvoffEl2, CurrentEl, SpsrEl1, ElrEl1, @@ -606,6 +619,10 @@ pub enum SystemRegister { /// Apple IMP-DEF configuration register the guest reads back, stored in /// `System::apple[slot]`. Apple(u8), + /// Apple IMP-DEF register of the banks the SPTM monitor configures (`op1` 4 to 6, and the + /// `op1` 0, 1 and 3 registers in `MONITOR_LOW_BANKS`), which + /// the SPTM monitor reads back; stored in `Cpu::apple_bank`. + AppleBank(u16), /// PAuth key register `APxAKey{Lo,Hi}_EL1`, stored in `System::pac_keys[slot]`. PacKey(u8), } @@ -1090,6 +1107,46 @@ fn simd_struct(word: u32) -> Result, DecodeErro } /// Number of Apple IMP-DEF configuration registers `System::apple` holds. pub const APPLE_SLOTS: usize = 21; +/// Number of `S3__c15` registers `Cpu::apple_bank` holds: eight banks of +/// `crm` x `op2` (16 x 8). +pub const APPLE_BANK_SLOTS: usize = 8 * 128; + +/// The `Cpu::apple_bank` slot of `S3__c15_c_`. +pub const fn apple_bank_slot(op1: u32, crm: u32, op2: u32) -> usize { + ((op1 as usize) << 7) | ((crm as usize) << 3) | op2 as usize +} + +/// The `S3_{0,1,3}_c15` registers (`op1`, `crm`, `op2`) the SPTM monitor's code accesses +/// that the kernel-side table above does not name, found by scanning its `mrs`/`msr` +/// words. Their roles are not identified; they are stored so the monitor reads back what +/// it wrote. Other registers in those banks stay unsupported. +const MONITOR_LOW_BANKS: [(u32, u32, u32); 25] = [ + (0, 0, 0), + (0, 1, 0), + (0, 1, 2), + (0, 1, 3), + (0, 2, 0), + (0, 2, 1), + (0, 3, 0), + (0, 4, 0), + (0, 4, 1), + (0, 5, 0), + (0, 6, 0), + (0, 7, 0), + (0, 9, 0), + (0, 9, 1), + (0, 10, 1), + (0, 11, 0), + (0, 11, 2), + (0, 14, 0), + (0, 15, 0), + (0, 15, 2), + (0, 15, 5), + (1, 8, 2), + (3, 8, 0), + (3, 9, 0), + (3, 10, 0), +]; /// The `S3_*` implementation-defined registers the M1 kernel and PMU driver touch /// at EL1 (`arch/arm64/include/asm/apple_m1_pmu.h`, `irq-apple-aic.c`). Configuration @@ -1126,9 +1183,16 @@ fn apple_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option Apple(19), (7, 0, 4) => Apple(20), (7, 6, 4) => AppleZero, - // The S3_5_c15_cN_0 state registers early init reads, none named in the XNU - // source tree. Read as zero until their roles are identified. + // The S3_5_c15 state registers the kernel reads during early boot. None are named + // in the XNU source tree that is available here, so they read as zero and writes + // are dropped until their roles are identified. (5, 2..=7, 0) => AppleZero, + // The EL2 and guarded-level banks the SPTM monitor configures: stored, so a + // read-modify-write or a read-back sees what was written. + (4..=6, _, _) => AppleBank(apple_bank_slot(op1, crm, op2) as u16), + _ if MONITOR_LOW_BANKS.contains(&(op1, crm, op2)) => { + AppleBank(apple_bank_slot(op1, crm, op2) as u16) + } _ => return None, }) } @@ -1190,6 +1254,19 @@ fn system_register( 0x0410 => SpEl0, 0x4410 => SpEl1, 0x6410 => SpEl2, + 0x4100 => SctlrEl2, + 0x4202 => TcrEl2, + 0x4200 => Ttbr0El2, + 0x4201 => Ttbr1El2, + 0x4a20 => MairEl2, + 0x4c00 => VbarEl2, + 0x4401 => ElrEl2, + 0x4400 => SpsrEl2, + 0x4520 => EsrEl2, + 0x4600 => FarEl2, + 0x4110 => HcrEl2, + 0x4d02 => TpidrEl2, + 0x4e03 => CntvoffEl2, 0x0422 => CurrentEl, 0x0400 => SpsrEl1, 0x0401 => ElrEl1, @@ -1264,8 +1341,8 @@ fn system_register( pub fn decode(word: u32) -> Result { use DecodeError::UnsupportedInstruction; use Instruction::*; - // PAuth instructions, hint-space forms included, run on the host. - if super::pauth::is_supported(word) { + // PAuth and guarded-execution instructions, hint-space forms included, run on the host. + if super::pauth::is_supported(word) || super::gxf::is_supported(word) { return Ok(Host(word)); } let width = width(word); diff --git a/src/emulator/src/aarch64/exception.rs b/src/emulator/src/aarch64/exception.rs index bc9ad9c..72657c5 100644 --- a/src/emulator/src/aarch64/exception.rs +++ b/src/emulator/src/aarch64/exception.rs @@ -1,4 +1,4 @@ -use super::cpu::Cpu; +use super::cpu::{Cpu, HCR_TGE}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] @@ -38,41 +38,57 @@ pub mod status { } pub fn take(cpu: &mut Cpu, reason: Reason, fault_address: u64) -> u64 { let from = cpu.system.el; - cpu.system.spsr_el1 = (u64::from(cpu.flags) & 0xf000_0000) + let to = if from == 2 || (from == 0 && cpu.system.hcr_el2 & HCR_TGE != 0) { + 2 + } else { + 1 + }; + let spsr = (u64::from(cpu.flags) & 0xf000_0000) | (cpu.system.pan << 22) | ((cpu.system.daif & 15) << 6) | (u64::from(from & 3) << 2) | u64::from(cpu.system.spsel); - cpu.system.elr_el1 = cpu.pc; - cpu.system.esr_el1 = (u64::from(reason.ec & 63) << 26) + let esr = (u64::from(reason.ec & 63) << 26) | (u64::from(reason.instruction) << 25) | u64::from(reason.status & 63) | u64::from(reason.iss & 0x1ff_ffff); - cpu.system.far_el1 = fault_address; - let group = if from != 1 { + if to == 2 { + cpu.system.spsr_el2 = spsr; + cpu.system.elr_el2 = cpu.pc; + cpu.system.esr_el2 = esr; + cpu.system.far_el2 = fault_address; + } else { + cpu.system.spsr_el1 = spsr; + cpu.system.elr_el1 = cpu.pc; + cpu.system.esr_el1 = esr; + cpu.system.far_el1 = fault_address; + } + // Same-EL entries pick the vector group by SPSel (SP_EL0 or SP_ELx); entries from a + // lower EL use the lower-EL groups. + let group = if from != to { 2 } else if cpu.system.spsel { 1 } else { 0 }; - let vector = cpu - .system - .vbar_el1 - .wrapping_add(group * 0x200) - .wrapping_add(reason.kind as u64 * 0x80); - let live_stack = if from == 1 && !cpu.system.spsel { - 0 + let vbar = if to == 2 { + cpu.system.vbar_el2 } else { - from & 3 + cpu.system.vbar_el1 }; + let vector = vbar + .wrapping_add(group * 0x200) + .wrapping_add(reason.kind as u64 * 0x80); + // The stack live at the exception is banked: SP_ELx when SPSel was set, else SP_EL0. + let live_stack = if cpu.system.spsel { from & 3 } else { 0 }; cpu.system.sp_el[live_stack as usize] = cpu.sp; - cpu.system.el = 1; + cpu.system.el = to; cpu.system.spsel = true; - cpu.sp = cpu.system.sp_el[1]; + cpu.sp = cpu.system.sp_el[to as usize]; cpu.system.daif = 15; - // SCTLR_EL1.SPAN clear: entry to EL1 sets PAN. - if cpu.system.sctlr_el1 & (1 << 23) == 0 { + // SCTLR_EL1.SPAN clear: entry to EL1 sets PAN. PAN belongs to EL1 only. + if to == 1 && cpu.system.sctlr_el1 & (1 << 23) == 0 { cpu.system.pan = 1; } cpu.monitor_valid = false; @@ -81,23 +97,24 @@ pub fn take(cpu: &mut Cpu, reason: Reason, fault_address: u64) -> u64 { cpu.pc } pub fn eret(cpu: &mut Cpu) -> u64 { - let spsr = cpu.system.spsr_el1; - let from_stack = if cpu.system.el == 1 && !cpu.system.spsel { - 0 + let el = cpu.system.el; + let (spsr, elr) = if el == 2 { + (cpu.system.spsr_el2, cpu.system.elr_el2) } else { - cpu.system.el & 3 + (cpu.system.spsr_el1, cpu.system.elr_el1) }; - cpu.system.sp_el[from_stack as usize] = cpu.sp; + let live_stack = if cpu.system.spsel { el & 3 } else { 0 }; + cpu.system.sp_el[live_stack as usize] = cpu.sp; cpu.monitor_valid = false; let to = ((spsr >> 2) & 3) as u8; cpu.system.el = to; - cpu.system.spsel = to == 1 && spsr & 1 != 0; - let live_stack = if to == 1 && !cpu.system.spsel { 0 } else { to }; - cpu.sp = cpu.system.sp_el[live_stack as usize]; + cpu.system.spsel = to != 0 && spsr & 1 != 0; + let restored = if cpu.system.spsel { to } else { 0 }; + cpu.sp = cpu.system.sp_el[restored as usize]; cpu.system.daif = (spsr >> 6) & 15; cpu.system.pan = (spsr >> 22) & 1; cpu.flags = (spsr & 0xf000_0000) as u32; - cpu.pc = cpu.system.elr_el1; + cpu.pc = elr; cpu.pc } @@ -183,6 +200,77 @@ mod tests { assert_eq!(cpu.system.el, 0); assert_eq!(cpu.sp, 0x2000); } + #[test] + fn exception_from_el2_stays_in_el2_and_banks_only_el2_state() { + let mut cpu = Cpu::default(); + cpu.system.el = 2; + cpu.system.spsel = true; + cpu.sp = 0x7000; + cpu.pc = 0x1234; + cpu.system.vbar_el2 = 0x20000; + cpu.system.vbar_el1 = 0x10000; + cpu.system.elr_el1 = 0xaaaa; + cpu.system.spsr_el1 = 0xbbbb; + // Same EL with SPSel set: the SP_EL2 group, synchronous. + assert_eq!(take(&mut cpu, Reason::default(), 0x55), 0x20200); + assert_eq!(cpu.system.el, 2); + assert_eq!(cpu.system.elr_el2, 0x1234); + assert_eq!(cpu.system.far_el2, 0x55); + assert_eq!((cpu.system.spsr_el2 >> 2) & 3, 2); + assert_eq!(cpu.system.elr_el1, 0xaaaa); + assert_eq!(cpu.system.spsr_el1, 0xbbbb); + assert_eq!(cpu.sp, 0x7000); + } + #[test] + fn eret_from_el2_returns_to_el1h_on_its_stack() { + let mut cpu = Cpu::default(); + cpu.system.el = 2; + cpu.system.spsel = true; + cpu.sp = 0x7000; + cpu.system.sp_el[1] = 0x9000; + cpu.system.spsr_el2 = (1 << 2) | 1; + cpu.system.elr_el2 = 0x4444; + assert_eq!(eret(&mut cpu), 0x4444); + assert_eq!(cpu.system.el, 1); + assert!(cpu.system.spsel); + assert_eq!(cpu.sp, 0x9000); + assert_eq!(cpu.system.sp_el[2], 0x7000); + } + #[test] + fn el0_svc_routes_to_el2_when_hcr_el2_tge_is_set() { + let mut cpu = Cpu::default(); + cpu.system.el = 0; + cpu.system.spsel = false; + cpu.system.hcr_el2 = HCR_TGE; + cpu.pc = 0x1000; + cpu.system.vbar_el2 = 0x20000; + cpu.system.vbar_el1 = 0x10000; + let reason = Reason { + ec: 0x15, + ..Reason::default() + }; + // Lower EL, AArch64, synchronous. + assert_eq!(take(&mut cpu, reason, 0), 0x20400); + assert_eq!(cpu.system.el, 2); + assert_eq!(cpu.system.elr_el2, 0x1000); + assert_eq!(cpu.system.elr_el1, 0); + assert_eq!(cpu.system.esr_el2 >> 26, 0x15); + } + #[test] + fn el0_svc_still_routes_to_el1_without_tge() { + let mut cpu = Cpu::default(); + cpu.system.el = 0; + cpu.system.spsel = false; + cpu.system.vbar_el2 = 0x20000; + cpu.system.vbar_el1 = 0x10000; + let reason = Reason { + ec: 0x15, + ..Reason::default() + }; + assert_eq!(take(&mut cpu, reason, 0), 0x10400); + assert_eq!(cpu.system.el, 1); + assert_eq!(cpu.system.elr_el2, 0); + } } #[cfg(test)] diff --git a/src/emulator/src/aarch64/gxf.rs b/src/emulator/src/aarch64/gxf.rs new file mode 100644 index 0000000..7b3f6c4 --- /dev/null +++ b/src/emulator/src/aarch64/gxf.rs @@ -0,0 +1,74 @@ +//! Guarded execution (GXF): `genter` and `gexit`, the instructions that move between +//! EL and the guarded levels GL1 and GL2 (Asahi Linux documentation, "Apple Proprietary +//! Instructions" and "SPRR and GXF"). +//! +//! The feature is off by default and only a monitor firmware (SPTM or TXM) enables it, +//! which this model does not run. Its enable bit reads as zero, so both instructions +//! take an undefined-instruction exception to EL1. The documentation says they "fault by +//! default" without naming the exception class; undefined-instruction is an assumption. + +use super::{Cpu, exception}; + +/// `gexit`: leave guarded mode. +const GEXIT: u32 = 0x0020_1400; +/// `genter` with its `imm5` in bits 4:0 (stored in `ESR_GLx[5:0]`). +const GENTER: u32 = 0x0020_1420; + +/// Whether `word` is `genter` or `gexit`. +pub fn is_supported(word: u32) -> bool { + word == GEXIT || word & !0x1f == GENTER +} + +/// Take the undefined-instruction exception `genter` or `gexit` raises while GXF is off. +/// `cpu.pc` is the address of the next instruction; the exception returns to the +/// instruction itself. +pub fn run(cpu: &mut Cpu, word: u32) { + debug_assert!(is_supported(word)); + cpu.pc = cpu.pc.wrapping_sub(4); + let pc = cpu.pc; + exception::take( + cpu, + exception::Reason { + ec: 0, + instruction: true, + status: 0, + ..Default::default() + }, + pc, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn genter_and_gexit_are_recognised_with_any_genter_immediate() { + assert!(is_supported(GEXIT)); + assert!(is_supported(GENTER)); + assert!(is_supported(GENTER | 0x1f)); + assert!(!is_supported(0xd503_201f)); + assert!(!is_supported(GEXIT + 4)); + } + + #[test] + fn with_guarded_execution_off_genter_is_an_undefined_instruction_at_the_instruction() { + let mut cpu = Cpu::default(); + cpu.system.el = 1; + cpu.system.spsel = true; + cpu.system.vbar_el1 = 0x4000_1000; + // The executor runs after the instruction at 0x4000_0000 has advanced `pc`. + cpu.pc = 0x4000_0004; + run(&mut cpu, GENTER); + assert_eq!( + cpu.system.elr_el1, 0x4000_0000, + "returns to the genter itself" + ); + assert_eq!(cpu.system.esr_el1 >> 26, 0, "EC is unknown reason"); + assert_eq!( + cpu.pc, + 0x4000_1000 + 0x200, + "current EL with SP_ELx, synchronous" + ); + } +} diff --git a/src/emulator/src/aarch64/host.rs b/src/emulator/src/aarch64/host.rs index 3946e30..291d488 100644 --- a/src/emulator/src/aarch64/host.rs +++ b/src/emulator/src/aarch64/host.rs @@ -2,11 +2,14 @@ //! and traps; the machine (or user mode) then runs the word on the host here. //! Floating point and the wider integer SIMD families live in [`fp`] and [`simd`]. -use super::{Cpu, fp, pauth, simd}; +use super::{Cpu, fp, gxf, pauth, simd}; /// Whether [`run`] implements `word`. pub fn is_supported(word: u32) -> bool { - fp::is_supported(word) || simd::is_supported(word) || pauth::is_supported(word) + fp::is_supported(word) + || simd::is_supported(word) + || pauth::is_supported(word) + || gxf::is_supported(word) } /// Run `word` on `cpu`. `word` must satisfy [`is_supported`]. @@ -15,6 +18,8 @@ pub fn run(cpu: &mut Cpu, word: u32) { fp::run(cpu, word) } else if simd::is_supported(word) { simd::run(cpu, word) + } else if gxf::is_supported(word) { + gxf::run(cpu, word) } else { pauth::run(cpu, word) } diff --git a/src/emulator/src/aarch64/mod.rs b/src/emulator/src/aarch64/mod.rs index f6c91c1..f284009 100644 --- a/src/emulator/src/aarch64/mod.rs +++ b/src/emulator/src/aarch64/mod.rs @@ -5,6 +5,7 @@ pub mod cpu; pub mod decode; pub mod exception; pub mod fp; +pub mod gxf; pub mod host; pub mod identification; pub mod pauth; diff --git a/src/emulator/src/aarch64/translate.rs b/src/emulator/src/aarch64/translate.rs index abba353..95ad5ea 100644 --- a/src/emulator/src/aarch64/translate.rs +++ b/src/emulator/src/aarch64/translate.rs @@ -27,21 +27,65 @@ pub struct Entry { pub ap: u8, pub executable: bool, } +/// The translation regime for the current EL. `split` regimes (EL1&0, and EL2 with +/// `HCR_EL2.E2H`, or EL0 under TGE with E2H) walk a TTBR0 and a TTBR1 range under the +/// `TCR_EL1` layout. EL2 without E2H has one TTBR0 range under `TCR_EL2`. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct Context { +struct Regime { sctlr: u64, tcr: u64, ttbr0: u64, ttbr1: u64, + split: bool, + /// Bit position of the physical-address size field: IPS (34:32) or PS (18:16). + ips_shift: u32, +} +impl Regime { + fn of(cpu: &Cpu) -> Self { + use super::cpu::{HCR_E2H, HCR_TGE}; + let hcr = cpu.system.hcr_el2; + let el2_regime = + cpu.system.el == 2 || (cpu.system.el == 0 && hcr & HCR_TGE != 0 && hcr & HCR_E2H != 0); + if !el2_regime { + return Self { + sctlr: cpu.system.sctlr_el1, + tcr: cpu.system.tcr_el1, + ttbr0: cpu.system.ttbr0_el1, + ttbr1: cpu.system.ttbr1_el1, + split: true, + ips_shift: 32, + }; + } + if hcr & HCR_E2H != 0 { + Self { + sctlr: cpu.system.sctlr_el2, + tcr: cpu.system.tcr_el2, + ttbr0: cpu.system.ttbr0_el2, + ttbr1: cpu.system.ttbr1_el2, + split: true, + ips_shift: 32, + } + } else { + Self { + sctlr: cpu.system.sctlr_el2, + tcr: cpu.system.tcr_el2, + ttbr0: cpu.system.ttbr0_el2, + ttbr1: 0, + split: false, + ips_shift: 16, + } + } + } +} +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Context { + regime: Regime, el: u8, } impl Context { fn of(cpu: &Cpu) -> Self { Self { - sctlr: cpu.system.sctlr_el1, - tcr: cpu.system.tcr_el1, - ttbr0: cpu.system.ttbr0_el1, - ttbr1: cpu.system.ttbr1_el1, + regime: Regime::of(cpu), el: cpu.system.el, } } @@ -85,10 +129,11 @@ impl Tlb { } } fn enabled(cpu: &Cpu) -> Result { - if cpu.system.sctlr_el1 & 1 == 0 { + let regime = Regime::of(cpu); + if regime.sctlr & 1 == 0 { return Ok(false); } - if cpu.system.tcr_el1 & 63 == 0 { + if regime.tcr & 63 == 0 { return Err(TranslateError::MalformedTables); } Ok(true) @@ -97,7 +142,10 @@ pub fn physical_bits(cpu: &Cpu) -> Result { if !enabled(cpu)? { return Ok(64); } - Ok([32, 36, 40, 42, 48, 52, 56, 56][((cpu.system.tcr_el1 >> 32) & 7) as usize]) + let regime = Regime::of(cpu); + // IPS (TCR_EL1 bits 34:32) or PS (TCR_EL2 bits 18:16). + let field = (regime.tcr >> regime.ips_shift) & 7; + Ok([32, 36, 40, 42, 48, 52, 56, 56][field as usize]) } fn permitted(ap: u8, executable: bool, access: Access, el: u8) -> Result<(), TranslateError> { if (el == 0 && ap & 1 == 0) @@ -119,8 +167,9 @@ pub fn translate( if !enabled(cpu)? { return Ok(virtual_address); } - let t1sz = ((cpu.system.tcr_el1 >> 16) & 63) as u32; - if t1sz == 0 { + let regime = Regime::of(cpu); + let t1sz = ((regime.tcr >> 16) & 63) as u32; + if regime.split && t1sz == 0 { return Err(TranslateError::MalformedTables); } if ((virtual_address >> 55) & 1 != 0 && virtual_address >> 48 != 0xffff) @@ -128,11 +177,11 @@ pub fn translate( { return Err(TranslateError::TranslationFault); } - let lower_bits = 64 - (cpu.system.tcr_el1 & 63) as u32; + let lower_bits = 64 - (regime.tcr & 63) as u32; let upper_bits = 64 - t1sz; let upper = if virtual_address >> lower_bits == 0 { false - } else if virtual_address >> upper_bits == u64::MAX >> upper_bits { + } else if regime.split && virtual_address >> upper_bits == u64::MAX >> upper_bits { true } else { return Err(TranslateError::TranslationFault); @@ -171,8 +220,9 @@ fn walk( access: Access, upper: bool, ) -> Result { - let tcr = cpu.system.tcr_el1; - if (upper && tcr & (1 << 23) != 0) || (!upper && tcr & (1 << 7) != 0) { + let regime = Regime::of(cpu); + let tcr = regime.tcr; + if (upper && tcr & (1 << 23) != 0) || (!upper && regime.split && tcr & (1 << 7) != 0) { return Err(TranslateError::TranslationFault); } let granule = if upper { @@ -184,11 +234,7 @@ fn walk( return Err(TranslateError::MalformedTables); } // ASID is in TTBR bits 63:48, never part of a physical table address. - let root = if upper { - cpu.system.ttbr1_el1 - } else { - cpu.system.ttbr0_el1 - }; + let root = if upper { regime.ttbr1 } else { regime.ttbr0 }; let va_bits = 64 - if upper { (tcr >> 16) & 63 } else { tcr & 63 }; if va_bits > 48 || va_bits <= u64::from(granule) { return Err(TranslateError::MalformedTables); @@ -694,4 +740,34 @@ mod tests { Err(TranslateError::PermissionFault) ); } + #[test] + fn el2_walks_only_ttbr0_el2_under_tcr_el2_and_has_no_upper_range() { + let mut cpu = Cpu::default(); + cpu.system.el = 2; + cpu.system.sctlr_el2 = 1; + // T0SZ = 16 (48-bit VAs), TG0 = 4 KiB, PS = 40 bits. + cpu.system.tcr_el2 = 16 | (2 << 16); + cpu.system.ttbr0_el2 = 0; + // The EL1&0 regime is disabled: EL2 translates from its own registers. + cpu.system.sctlr_el1 = 0; + let mut memory = Memory::new(); + memory.chain(0, 0, 12, 4, 0, 0x4000_0443); + let mut tlb = Tlb::new(); + assert_eq!( + translate(&mut tlb, &cpu, &mut memory, 0, Access::Read), + Ok(0x4000_0000) + ); + assert_eq!(physical_bits(&cpu), Ok(40)); + // Bit 55 set with an all-ones upper half: a TTBR1 address, which EL2 lacks. + assert_eq!( + translate( + &mut tlb, + &cpu, + &mut memory, + 0xffff_8000_0000_0000, + Access::Read + ), + Err(TranslateError::TranslationFault) + ); + } } diff --git a/src/emulator/src/system/m1.rs b/src/emulator/src/system/m1.rs index 2b7266b..09cb468 100644 --- a/src/emulator/src/system/m1.rs +++ b/src/emulator/src/system/m1.rs @@ -36,9 +36,9 @@ pub const DEFAULT_CMDLINE: &str = "earlycon nokaslr loglevel=8"; /// `MIDR_EL1` of an Icestorm core: implementer Apple (0x61), architecture from ID /// registers (0xf), part 0x022. pub const MIDR_EL1: u64 = 0x610f_0220; -/// `ID_AA64PFR0_EL1`: EL0 and EL1 in AArch64, no EL2 or EL3, and floating point and -/// Advanced SIMD implemented, as on the M1. -pub const ID_AA64PFR0_EL1: u64 = 0x11; +/// `ID_AA64PFR0_EL1`: EL0, EL1 and EL2 in AArch64 (EL2 hosts the SPTM monitor, which +/// `CurrentEl` must report), no EL3, and floating point and Advanced SIMD implemented. +pub const ID_AA64PFR0_EL1: u64 = 0x111; /// `ID_AA64MMFR0_EL1`: 4 KiB and 16 KiB translation granules (the Asahi kernel is built /// for 16 KiB pages and refuses a CPU without one), no 64 KiB granule, 40-bit PA. pub const ID_AA64MMFR0_EL1: u64 = (0xf << 24) | (1 << 20) | 2; @@ -51,11 +51,16 @@ pub const ID_AA64ISAR1_EL1: u64 = (1 << 20) | (1 << 24) | (1 << 4); const CPUS: u32 = 1; /// Give `system` the identity of an M1 core. -pub fn identify(system: &mut crate::aarch64::cpu::System) { +pub fn identify(cpu: &mut crate::aarch64::cpu::Cpu) { + let system = &mut cpu.system; system.midr_el1 = MIDR_EL1; system.id_aa64pfr0_el1 = ID_AA64PFR0_EL1; system.id_aa64mmfr0_el1 = ID_AA64MMFR0_EL1; system.id_aa64isar1_el1 = ID_AA64ISAR1_EL1; + // The SPTM monitor polls `S3_6_c15_c12_4` until bit 0 is set before it continues + // (`mrs x0, s3_6_c15_c12_4; and x1, x0, #1; cbz x1, `). Nothing here names the + // register, so bit 0 reads as set, which is the only value that lets the loop exit. + cpu.apple_bank.0[crate::aarch64::decode::apple_bank_slot(6, 12, 4)] = 1; } /// The SoC devices on the bus: the AIC, the power-state blocks, the console, and the diff --git a/src/emulator/src/system/machine.rs b/src/emulator/src/system/machine.rs index bfebffa..0e2171f 100644 --- a/src/emulator/src/system/machine.rs +++ b/src/emulator/src/system/machine.rs @@ -196,6 +196,92 @@ pub struct Machine { pub blocks_run: u64, faulted: bool, device_access: bool, + /// Per-instruction observer installed by [`Machine::set_step_trace`]. + step_trace: Option>, + /// Per-access observer installed by [`Machine::set_memory_trace`]. + memory_trace: Option>, + /// The step in progress: its start `pc` and the system registers before it ran. + step_pending: Option<(u64, crate::aarch64::cpu::System)>, +} + +/// One step seen by a [`Machine::set_step_trace`] observer. Single-stepping makes each +/// step one instruction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Step { + /// Address the step started at. + pub pc: u64, + /// Address after the step. + pub next_pc: u64, + /// Each watched system register that changed, as `(name, before, after)`. + pub changes: Vec<(&'static str, u64, u64)>, +} + +/// One load or store seen by a [`Machine::set_memory_trace`] observer, after translation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MemoryAccess { + /// Address of the instruction that made the access. + pub pc: u64, + pub virtual_address: u64, + pub physical: u64, + /// Bytes accessed: 1, 2, 4, 8 or 16. + pub width: u8, + pub write: bool, + /// The low 8 bytes stored; 0 for a load, whose value is not known before it runs. + pub value: u64, +} + +/// The watched system registers whose values differ between `before` and `after`. +fn system_changes( + before: &crate::aarch64::cpu::System, + after: &crate::aarch64::cpu::System, +) -> Vec<(&'static str, u64, u64)> { + macro_rules! watch { + ($($field:ident),* $(,)?) => {{ + let mut changes = Vec::new(); + $( + if before.$field != after.$field { + changes.push((stringify!($field), before.$field as u64, after.$field as u64)); + } + )* + changes + }}; + } + let mut changes = watch!( + el, + spsel, + daif, + pan, + spsr_el1, + elr_el1, + esr_el1, + far_el1, + vbar_el1, + sctlr_el1, + ttbr0_el1, + ttbr1_el1, + tcr_el1, + mair_el1, + spsr_el2, + elr_el2, + esr_el2, + far_el2, + vbar_el2, + sctlr_el2, + ttbr0_el2, + ttbr1_el2, + tcr_el2, + mair_el2, + hcr_el2, + tpidr_el2, + cntvoff_el2, + ); + const SP_NAMES: [&str; 4] = ["sp_el0", "sp_el1", "sp_el2", "sp_el3"]; + for (index, name) in SP_NAMES.iter().enumerate() { + if before.sp_el[index] != after.sp_el[index] { + changes.push((name, before.sp_el[index], after.sp_el[index])); + } + } + changes } impl Machine { pub fn new(memory: M) -> Self { @@ -231,6 +317,9 @@ impl Machine { blocks_run: 0, faulted: false, device_access: false, + step_trace: None, + memory_trace: None, + step_pending: None, } } /// A vCPU whose translated blocks are shared with other vCPUs. @@ -282,15 +371,68 @@ impl Machine { && self.timer_due() }) } + /// Installs a per-instruction trace. While set, blocks are compiled one instruction at + /// a time, and `sink` sees each step with the watched system registers it changed. + /// `None` removes the trace. + pub fn set_step_trace(&mut self, sink: Option>) { + self.step_trace = sink; + self.rebuild_cache(); + } + /// Installs a per-access trace: `sink` sees every load and store the guest makes, + /// translated, except atomics and SIMD structure accesses. While set, the inline data + /// TLB is off so every access reaches the host. `None` removes the trace. + pub fn set_memory_trace(&mut self, sink: Option>) { + self.memory_trace = sink; + self.rebuild_cache(); + } + /// A fresh block cache for the trace state: single-step blocks while stepping, no inline + /// data TLB while either trace is set, and the default cache otherwise. Compiled blocks + /// and background compilation of the previous cache are dropped. + fn rebuild_cache(&mut self) { + let traced = self.step_trace.is_some() || self.memory_trace.is_some(); + let inline = !traced && !std::env::var_os("VOLT_INLINE_MEMORY").is_some_and(|v| v == "0"); + self.cache = if self.step_trace.is_some() { + Cache::single_step(false) + } else if inline { + Cache::with_inline_memory() + } else { + Cache::new() + }; + self.step_pending = None; + } pub fn run(&mut self, serial: &mut Pl011, gic: &mut Gicv2) -> Result { self.run_with_devices(&mut (serial, gic)) } + /// Reports the step in progress, if any, with the registers it changed. A step ends + /// when the next one starts or the run returns, so an exception entry the step + /// caused is part of it. + fn flush_step(&mut self) { + if let Some((pc, before)) = self.step_pending.take() + && let Some(sink) = self.step_trace.as_mut() + { + sink(&Step { + pc, + next_pc: self.cpu.pc, + changes: system_changes(&before, &self.cpu.system), + }); + } + } pub fn run_with_devices(&mut self, devices: &mut impl DeviceIo) -> Result { + let result = self.run_inner(devices); + self.flush_step(); + result + } + fn run_inner(&mut self, devices: &mut impl DeviceIo) -> Result { self.stalled = None; for _ in 0..64 { if self.timer_edge() { return Ok(Exit::Timer); } + self.flush_step(); + self.step_pending = self + .step_trace + .is_some() + .then(|| (self.cpu.pc, self.cpu.system.clone())); let fiq = self.fiq_line || self.fiq_asserted(); self.cpu.system.isr_el1 = u64::from(self.irq_line) << 7 | u64::from(fiq) << 6; let kind = if fiq && self.cpu.system.daif & 1 == 0 { @@ -662,12 +804,13 @@ impl Machine { } /// A synchronous abort with fault status `status`. fn abort_with_status(&mut self, address: u64, status: u8, instruction: bool, write: bool) { - let ec = if instruction { - if self.cpu.system.el == 1 { 0x21 } else { 0x20 } - } else if self.cpu.system.el == 1 { - 0x25 - } else { - 0x24 + // EC 0x20/0x24 for a lower-EL abort, 0x21/0x25 for one taken from EL1 or EL2. + let same_el = self.cpu.system.el != 0; + let ec = match (instruction, same_el) { + (true, false) => 0x20, + (true, true) => 0x21, + (false, false) => 0x24, + (false, true) => 0x25, }; self.cpu.trap = Trap::None; if !instruction { @@ -953,6 +1096,16 @@ impl Machine { return Ok(()); } }; + if let Some(sink) = self.memory_trace.as_mut() { + sink(&MemoryAccess { + pc: self.cpu.pc.wrapping_sub(4), + virtual_address: address, + physical, + width, + write: !load, + value: if load { 0 } else { value }, + }); + } let mut bytes = [0u8; 16]; if !load { if self.cpu.vector_dest { @@ -1362,6 +1515,98 @@ mod tests { "ELR is the BRK itself" ); } + + #[test] + fn el2_with_e2h_sends_el1_register_names_to_the_el2_banks() { + let mut machine = machine(); + machine.cpu.system.el = 2; + machine.cpu.system.spsel = true; + machine.cpu.system.hcr_el2 = crate::aarch64::cpu::HCR_E2H; + machine.cpu.system.vbar_el1 = 0x1111; + machine.cpu.system.vbar_el2 = 0x2222; + // movz x0, #0x1234; msr vbar_el1, x0; mrs x1, vbar_el1 + run_to_hvc(&mut machine, &[0xd2824680, 0xd518c000, 0xd538c001, HVC]); + assert_eq!(machine.cpu.x[1], 0x1234); + assert_eq!(machine.cpu.system.vbar_el2, 0x1234); + assert_eq!(machine.cpu.system.vbar_el1, 0x1111); + } + + #[test] + fn el2_without_e2h_writes_the_el1_bank_and_leaves_el2_alone() { + let mut machine = machine(); + machine.cpu.system.el = 2; + machine.cpu.system.spsel = true; + machine.cpu.system.hcr_el2 = 0; + machine.cpu.system.vbar_el1 = 0x1111; + machine.cpu.system.vbar_el2 = 0x2222; + run_to_hvc(&mut machine, &[0xd2824680, 0xd518c000, 0xd538c001, HVC]); + assert_eq!(machine.cpu.x[1], 0x1234); + assert_eq!(machine.cpu.system.vbar_el1, 0x1234); + assert_eq!(machine.cpu.system.vbar_el2, 0x2222); + } + + #[test] + fn step_trace_reports_each_instruction_and_the_system_register_it_changed() { + use parking_lot::Mutex; + use std::sync::Arc; + let mut machine = machine(); + machine.cpu.system.el = 1; + machine.cpu.system.spsel = true; + let steps = Arc::new(Mutex::new(Vec::new())); + let sink = steps.clone(); + machine.set_step_trace(Some(Box::new(move |step| { + sink.lock().push(step.clone()); + }))); + // movz x0, #1; msr vbar_el1, x0; hvc + run_to_hvc(&mut machine, &[0xd2800020, 0xd518c000, HVC]); + let steps = steps.lock(); + let pcs: Vec = steps.iter().map(|s| s.pc).collect(); + assert_eq!(pcs, [0x40000000, 0x40000004, 0x40000008]); + assert!(steps[0].changes.is_empty()); + assert_eq!(steps[1].changes, [("vbar_el1", 0, 1)]); + assert_eq!(steps[1].next_pc, 0x40000008); + } + #[test] + fn memory_trace_reports_each_translated_access_with_its_pc_and_stored_value() { + use parking_lot::Mutex; + use std::sync::Arc; + let mut machine = machine(); + machine.cpu.system.el = 1; + machine.cpu.system.spsel = true; + let accesses = Arc::new(Mutex::new(Vec::new())); + let sink = accesses.clone(); + machine.set_memory_trace(Some(Box::new(move |access| sink.lock().push(*access)))); + // movz x0,#0x1234; movz x1,#0x4000,lsl #16; add x1,x1,#0x800; str x0,[x1]; + // ldr w2,[x1,#4] + run_to_hvc( + &mut machine, + &[ + 0xd2824680, 0xd2a80001, 0x91200021, 0xf9000020, 0xb9400422, HVC, + ], + ); + let accesses = accesses.lock(); + assert_eq!( + *accesses, + [ + MemoryAccess { + pc: 0x4000000c, + virtual_address: 0x40000800, + physical: 0x40000800, + width: 8, + write: true, + value: 0x1234, + }, + MemoryAccess { + pc: 0x40000010, + virtual_address: 0x40000804, + physical: 0x40000804, + width: 4, + write: false, + value: 0, + }, + ] + ); + } const LDXR_X1_X0: u32 = 0xc85f7c01; const STXR_W2_X3_X0: u32 = 0xc8027c03; const HVC: u32 = 0xd4000002; diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index e13c5d1..24378b2 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -9,6 +9,7 @@ pub mod esr; pub mod fdt; pub mod m1; pub mod machine; +pub mod monitor; pub mod psci; pub mod smp; pub mod xnu; @@ -511,11 +512,17 @@ pub fn boot_system( }); let mut machine = machine::Machine::new(shared_mem); if apple { - m1::identify(&mut machine.cpu.system); + m1::identify(&mut machine.cpu); } machine.cpu.pc = layout.entry; machine.cpu.x[0] = layout.boot_info; machine.cpu.x[1..4].fill(0); + if is_xnu && xnu::is_fileset(image) { + // A kernel collection enters at its reset trampoline: x0 is the reset type (0 for + // a cold boot) and x1 the boot arguments. + machine.cpu.x[0] = 0; + machine.cpu.x[1] = layout.boot_info; + } let started = Instant::now(); let mut exits = 0; let mut timer_interrupts = 0; diff --git a/src/emulator/src/system/monitor.rs b/src/emulator/src/system/monitor.rs new file mode 100644 index 0000000..7f414ac --- /dev/null +++ b/src/emulator/src/system/monitor.rs @@ -0,0 +1,558 @@ +//! Apple's GL2 monitor (SPTM), entered at EL2 with the MMU off. +//! +//! The boot firmware copies the monitor image into DRAM and jumps to its entry at the +//! image's *physical* address, with x0 the physical address of a boot structure. The +//! monitor builds its own translation tables (its `__BOOTDATA` and `__LATE_CONST` +//! segments are zero in the file), sets `HCR_EL2.{E2H,TGE}`, and turns the MMU on +//! itself. [`load`] does the same. The structure's fields come from the entry code and +//! from where the monitor reads its copy of it: +//! +//! | offset | value | +//! |--------|------------------------------------------------------------------------| +//! | 0 | count (the code tests `>= 3`) | +//! | 8 | virtual address of the first DRAM byte in the monitor's map (`x22`) | +//! | 16 | physical address of the first DRAM byte (`x23`) | +//! | 24 | bytes of DRAM to map (`x24`) | +//! | 0x60 | device tree address, as the monitor sees it (the linear map) | +//! | 0x68 | device tree size in bytes (32 bits) | +//! | 1152 | physical address of a scratch buffer, zeroed and stamped `"SPTM"` | +//! +//! The monitor maps all of DRAM linearly (`virtual = physical + [8] - [16]`) in 32 MiB +//! blocks, which drop the low 25 bits of both addresses, so both bases are block-aligned. +//! Its own image sits inside that map at its link address, which fixes `[8]`: the image's +//! physical base equals its virtual base modulo 32 MiB. Everything the monitor reads +//! before it has its own mappings (the boot structure, the device tree, the scratch +//! buffer, the kernel collection it protects) is reached through that map, so it all lies +//! in DRAM. Seen with the per-instruction and memory traces (`Machine::set_step_trace`, +//! `Machine::set_memory_trace`). + +use crate::{ + memory::{GuestMemory, MemoryError}, + system::xnu, +}; + +/// The monitor's block-mapping granularity: 32 MiB (a 16 KiB-granule level-2 block). +pub const BLOCK: u64 = 32 << 20; +/// Offsets of the device-tree address and size in the boot structure, read from the +/// monitor's copy of it (`ldr x9, [x8]; ldr w8, [x8, #8]` at `+0x60`). +const DEVICE_TREE_SLOT: u64 = 0x60; +/// Offset of the scratch-buffer pointer in the boot structure (`ldr x5, [x20, #1152]`). +const SCRATCH_SLOT: u64 = 1152; +/// Bytes the entry code zeroes at the scratch buffer (`mov x7, #0x17c0`). +const SCRATCH_BYTES: u64 = 0x17c0; +/// Bytes of boot structure written: through the scratch slot. +const BOOT_BYTES: usize = SCRATCH_SLOT as usize + 8; + +#[derive(Debug)] +pub enum Error { + /// The image is not a Mach-O executable the loader can place. + Image(crate::system::boot::Error), + Memory(MemoryError), + /// The physical image base is not congruent with the virtual base modulo 32 MiB. + Misaligned { + virtual_base: u64, + physical_base: u64, + }, + /// DRAM is not 32 MiB-aligned, or is smaller than one block. A size under one block + /// makes the monitor's mapping loop wrap and run for ever. + Dram { + base: u64, + size: u64, + }, + /// A part the monitor reads through its linear map is outside DRAM. + OutsideDram { + what: &'static str, + physical: u64, + }, + /// The device tree is not a well-formed Apple flattened device tree. + DeviceTree(&'static str), +} + +impl From for Error { + fn from(error: MemoryError) -> Self { + Self::Memory(error) + } +} + +/// Where the firmware puts each part of the monitor, all physical addresses. +#[derive(Clone, Copy, Debug)] +pub struct Layout { + /// First byte of DRAM and its size, both multiples of [`BLOCK`]. + pub dram_base: u64, + pub dram_size: u64, + /// Physical address of the image's lowest segment (`__TEXT`). Must be congruent + /// with that segment's link address modulo [`BLOCK`], and inside DRAM. + pub image_phys: u64, + /// Physical address of the boot structure, inside DRAM. + pub boot_phys: u64, + /// Physical address of the monitor's scratch buffer, inside DRAM. + pub scratch_phys: u64, + /// Physical address the device tree is copied to, inside DRAM. + pub device_tree_phys: u64, +} + +/// A monitor image placed in memory, ready to run. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Monitor { + /// Physical address of the entry point (`LC_UNIXTHREAD` pc, relocated). + pub entry: u64, + /// Physical address of the boot structure, passed in x0. + pub boot_phys: u64, +} + +/// Bit 31 of an ADT property length marks a placeholder property; the length is the low 31 +/// bits. The monitor adds the raw length to the property's address, so the flag would make +/// every walk overflow its bounds check. +const PLACEHOLDER: u32 = 1 << 31; + +/// A copy of the Apple flattened device tree `tree` with every placeholder flag cleared. +/// Nodes are `{properties: u32, children: u32}`, each property a 32-byte name, a `u32` +/// length and its value padded to four bytes, then the children. +pub fn without_placeholder_flags(tree: &[u8]) -> Result, Error> { + const TRUNCATED: Error = Error::DeviceTree("device tree is truncated"); + fn word(tree: &[u8], at: usize) -> Result { + let bytes = tree.get(at..at + 4).ok_or(TRUNCATED)?; + Ok(u32::from_le_bytes(bytes.try_into().unwrap())) + } + fn node(tree: &mut [u8], at: usize) -> Result { + let properties = word(tree, at)?; + let children = word(tree, at + 4)?; + let mut at = at + 8; + for _ in 0..properties { + let length = word(tree, at + 32)? & !PLACEHOLDER; + tree.get_mut(at + 32..at + 36) + .ok_or(TRUNCATED)? + .copy_from_slice(&length.to_le_bytes()); + at += 36 + (length as usize).next_multiple_of(4); + if at > tree.len() { + return Err(TRUNCATED); + } + } + for _ in 0..children { + at = node(tree, at)?; + } + Ok(at) + } + let mut copy = tree.to_vec(); + node(&mut copy, 0)?; + Ok(copy) +} + +/// Appends the property `name = {address: u64, size: u64}` to the device tree's +/// `chosen/memory-map` node, the way the boot firmware records a region it loaded (the +/// monitor looks up `BootKC-rs`, the kernel collection, and panics without it). +pub fn add_memory_map_region( + tree: &[u8], + name: &str, + address: u64, + size: u64, +) -> Result, Error> { + const TRUNCATED: Error = Error::DeviceTree("device tree is truncated"); + const NAME: usize = 32; + fn word(tree: &[u8], at: usize) -> Result { + let bytes = tree.get(at..at + 4).ok_or(TRUNCATED)?; + Ok(u32::from_le_bytes(bytes.try_into().unwrap())) + } + /// The offset just past the properties of the node at `at`, and its name. + fn properties(tree: &[u8], at: usize) -> Result<(usize, Option<&[u8]>), Error> { + let mut found = None; + let mut cursor = at + 8; + for _ in 0..word(tree, at)? { + let length = (word(tree, cursor + NAME)? & !PLACEHOLDER) as usize; + let value = cursor + NAME + 4; + if tree.get(cursor..cursor + 5) == Some(b"name\0") { + found = tree.get(value..value + length); + } + cursor = value + length.next_multiple_of(4); + if cursor > tree.len() { + return Err(TRUNCATED); + } + } + Ok((cursor, found.map(|n| n.strip_suffix(b"\0").unwrap_or(n)))) + } + /// The offset just past the node at `at` and everything under it. + fn skip(tree: &[u8], at: usize) -> Result { + let (mut cursor, _) = properties(tree, at)?; + for _ in 0..word(tree, at + 4)? { + cursor = skip(tree, cursor)?; + } + Ok(cursor) + } + /// The offset of the child of the node at `at` named `wanted`. + fn child(tree: &[u8], at: usize, wanted: &[u8]) -> Result { + let (mut cursor, _) = properties(tree, at)?; + for _ in 0..word(tree, at + 4)? { + if properties(tree, cursor)?.1 == Some(wanted) { + return Ok(cursor); + } + cursor = skip(tree, cursor)?; + } + Err(Error::DeviceTree( + "device tree has no chosen/memory-map node", + )) + } + let chosen = child(tree, 0, b"chosen")?; + let map = child(tree, chosen, b"memory-map")?; + let (end, _) = properties(tree, map)?; + let mut property = [0u8; NAME + 4 + 16]; + let name = name.as_bytes(); + if name.len() >= NAME { + return Err(Error::DeviceTree("property name is too long")); + } + property[..name.len()].copy_from_slice(name); + property[NAME..NAME + 4].copy_from_slice(&16u32.to_le_bytes()); + property[NAME + 4..NAME + 12].copy_from_slice(&address.to_le_bytes()); + property[NAME + 12..].copy_from_slice(&size.to_le_bytes()); + let mut out = Vec::with_capacity(tree.len() + property.len()); + out.extend_from_slice(&tree[..end]); + out.extend_from_slice(&property); + out.extend_from_slice(&tree[end..]); + let count = word(&out, map)? + 1; + out[map..map + 4].copy_from_slice(&count.to_le_bytes()); + Ok(out) +} + +/// Places the `MH_EXECUTE` monitor `image` and `device_tree`, and writes the boot structure. +pub fn load( + memory: &mut impl GuestMemory, + image: &[u8], + device_tree: &[u8], + layout: &Layout, +) -> Result { + let kernel = xnu::parse(image).map_err(Error::Image)?; + let device_tree = without_placeholder_flags(device_tree)?; + let base = kernel + .segments + .iter() + .filter(|s| s.vmsize != 0) + .map(|s| s.vmaddr) + .min() + .unwrap_or(0); + let (dram, size) = (layout.dram_base, layout.dram_size); + if dram & (BLOCK - 1) != 0 || size & (BLOCK - 1) != 0 || size < BLOCK { + return Err(Error::Dram { base: dram, size }); + } + if (base ^ layout.image_phys) & (BLOCK - 1) != 0 { + return Err(Error::Misaligned { + virtual_base: base, + physical_base: layout.image_phys, + }); + } + let end = kernel + .segments + .iter() + .map(|s| s.vmaddr + s.vmsize) + .max() + .unwrap_or(base); + for (what, physical, length) in [ + ("image", layout.image_phys, end - base), + ("boot structure", layout.boot_phys, BOOT_BYTES as u64), + ("scratch buffer", layout.scratch_phys, SCRATCH_BYTES), + ( + "device tree", + layout.device_tree_phys, + device_tree.len() as u64, + ), + ] { + if physical < dram || physical + length > dram + size { + return Err(Error::OutsideDram { what, physical }); + } + } + for segment in &kernel.segments { + if segment.vmsize == 0 { + continue; + } + let mut bytes = vec![0u8; segment.vmsize as usize]; + let file = &image[segment.fileoff as usize..][..segment.filesize as usize]; + bytes[..file.len()].copy_from_slice(file); + memory.write(layout.image_phys + (segment.vmaddr - base), &bytes)?; + } + // The image sits at its link address in the linear map, which fixes where DRAM starts. + let dram_va = base.wrapping_sub(layout.image_phys - dram); + let linear = dram_va.wrapping_sub(dram); + let mut boot = [0u8; BOOT_BYTES]; + let mut put = |at: u64, value: u64| { + boot[at as usize..at as usize + 8].copy_from_slice(&value.to_le_bytes()); + }; + put(0, 3); + put(8, dram_va); + put(16, dram); + put(24, size); + put( + DEVICE_TREE_SLOT, + layout.device_tree_phys.wrapping_add(linear), + ); + put(DEVICE_TREE_SLOT + 8, device_tree.len() as u64); + put(SCRATCH_SLOT, layout.scratch_phys); + memory.write(layout.device_tree_phys, &device_tree)?; + memory.write(layout.boot_phys, &boot)?; + memory.write(layout.scratch_phys, &vec![0u8; SCRATCH_BYTES as usize])?; + Ok(Monitor { + entry: layout.image_phys + (kernel.entry - base), + boot_phys: layout.boot_phys, + }) +} + +/// Sets the CPU to the state the firmware leaves for the monitor: EL2, MMU off, at the +/// entry's physical address, with the boot structure in x0. +pub fn prepare_cpu(cpu: &mut crate::aarch64::cpu::Cpu, monitor: &Monitor) { + let system = &mut cpu.system; + system.el = 2; + system.spsel = true; + // EL1 is AArch64. The monitor sets `E2H` and `TGE` itself. + system.hcr_el2 = 1 << 31; + system.sctlr_el2 = 0; + system.daif = 15; + cpu.pc = monitor.entry; + cpu.x[0] = monitor.boot_phys; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::memory::{PhysicalMemory, Region}; + + /// A minimal `MH_EXECUTE` arm64 image: one `__TEXT` segment holding `payload`, + /// linked at `vmaddr`, with an `LC_UNIXTHREAD` entry `entry_offset` bytes in. + fn image(vmaddr: u64, entry_offset: u64, payload: &[u8]) -> Vec { + let thread = 8 + 8 + 272; + let segment = 72; + let header = 32; + let file_size = 0x4000u64; + let mut out = Vec::new(); + for word in [ + 0xfeed_facf_u32, + 0x0100_000c, + 0, + 2, + 2, + (segment + thread) as u32, + 0, + 0, + ] { + out.extend(word.to_le_bytes()); + } + out.extend(0x19u32.to_le_bytes()); + out.extend((segment as u32).to_le_bytes()); + let mut name = [0u8; 16]; + name[..6].copy_from_slice(b"__TEXT"); + out.extend(name); + for word in [vmaddr, file_size, 0, file_size] { + out.extend(word.to_le_bytes()); + } + for word in [5u32, 5, 0, 0] { + out.extend(word.to_le_bytes()); + } + out.extend(5u32.to_le_bytes()); + out.extend((thread as u32).to_le_bytes()); + out.extend(6u32.to_le_bytes()); + out.extend(68u32.to_le_bytes()); + let mut state = [0u64; 34]; + state[32] = vmaddr + entry_offset; + for word in state { + out.extend(word.to_le_bytes()); + } + out.resize(header + segment + thread, 0); + out.resize(0x200, 0); + out.extend_from_slice(payload); + out.resize(file_size as usize, 0); + out + } + + const DRAM: u64 = 0x8_0000_0000; + const VMADDR: u64 = 0xffff_fff0_2700_4000; + + fn memory() -> PhysicalMemory { + PhysicalMemory::new(vec![ + Region::ram(DRAM, vec![0; (2 * BLOCK) as usize]).unwrap(), + ]) + .unwrap() + } + + /// Two blocks of DRAM, the image at its link offset in the first, the rest below it. + fn layout() -> Layout { + Layout { + dram_base: DRAM, + dram_size: 2 * BLOCK, + image_phys: DRAM + (VMADDR & (BLOCK - 1)), + boot_phys: DRAM + 0x10_0000, + scratch_phys: DRAM + 0x11_0000, + device_tree_phys: DRAM + 0x12_0000, + } + } + + #[test] + fn the_image_is_placed_and_the_boot_structure_describes_dram() { + let image = image(VMADDR, 0x200, &[0xaa, 0xbb, 0xcc, 0xdd]); + let mut memory = memory(); + let layout = layout(); + let monitor = load(&mut memory, &image, &[0; 8], &layout).unwrap(); + assert_eq!(monitor.entry, layout.image_phys + 0x200); + let mut word = [0u8; 8]; + let mut field = |at: u64| { + memory.read(layout.boot_phys + at, &mut word).unwrap(); + u64::from_le_bytes(word) + }; + // The image is at its link address in the linear map, so DRAM's first byte is + // mapped as many bytes below the link address as the image is above DRAM's start. + let dram_va = VMADDR - (layout.image_phys - DRAM); + assert_eq!(field(0), 3); + assert_eq!(field(8), dram_va); + assert_eq!(field(16), DRAM); + assert_eq!(field(24), 2 * BLOCK); + assert_eq!(field(SCRATCH_SLOT), layout.scratch_phys); + assert_eq!( + field(DEVICE_TREE_SLOT), + layout.device_tree_phys - DRAM + dram_va + ); + assert_eq!(field(DEVICE_TREE_SLOT + 8), 8); + let mut payload = [0u8; 4]; + memory + .read(layout.image_phys + 0x200, &mut payload) + .unwrap(); + assert_eq!(payload, [0xaa, 0xbb, 0xcc, 0xdd]); + assert_eq!( + dram_va & (BLOCK - 1), + 0, + "DRAM's virtual base is block-aligned" + ); + } + + #[test] + fn a_physical_base_that_is_not_congruent_with_the_virtual_base_is_refused() { + let layout = Layout { + image_phys: DRAM + 0x8000, + ..layout() + }; + assert!(matches!( + load(&mut memory(), &image(VMADDR, 0x200, &[]), &[0; 8], &layout), + Err(Error::Misaligned { .. }) + )); + } + + #[test] + fn dram_that_is_unaligned_or_under_one_block_is_refused() { + for (base, size) in [ + (DRAM + 0x1000, 2 * BLOCK), + (DRAM, BLOCK - 0x1000), + (DRAM, 0), + ] { + let layout = Layout { + dram_base: base, + dram_size: size, + ..layout() + }; + assert!(matches!( + load(&mut memory(), &image(VMADDR, 0x200, &[]), &[0; 8], &layout), + Err(Error::Dram { .. }) + )); + } + } + + #[test] + fn a_part_the_monitor_reads_through_its_map_must_lie_in_dram() { + let layout = Layout { + boot_phys: DRAM + 2 * BLOCK, + ..layout() + }; + assert!(matches!( + load(&mut memory(), &image(VMADDR, 0x200, &[]), &[0; 8], &layout), + Err(Error::OutsideDram { + what: "boot structure", + .. + }) + )); + } + + /// A root node with one property `a` of `length` (raw, flags included) holding `data`, + /// and one child with no properties. + fn tree(length: u32, data: &[u8]) -> Vec { + let mut out = Vec::new(); + out.extend(1u32.to_le_bytes()); + out.extend(1u32.to_le_bytes()); + let mut name = [0u8; 32]; + name[0] = b'a'; + out.extend(name); + out.extend(length.to_le_bytes()); + out.extend(data); + out.resize(out.len().next_multiple_of(4), 0); + out.extend([0u8; 8]); + out + } + + #[test] + fn placeholder_flags_are_cleared_from_property_lengths_and_nothing_else_changes() { + let flagged = tree(PLACEHOLDER | 5, b"hello"); + let clean = without_placeholder_flags(&flagged).unwrap(); + assert_eq!(clean, tree(5, b"hello")); + // The child node after the padded value is still found, so the walk stayed in step. + assert_eq!(clean.len(), flagged.len()); + } + + #[test] + fn a_tree_that_ends_inside_a_property_is_refused() { + let mut cut = tree(PLACEHOLDER | 5, b"hello"); + cut.truncate(40); + assert!(matches!( + without_placeholder_flags(&cut), + Err(Error::DeviceTree(_)) + )); + } + + /// A node: its properties (name, value) and children, in the Apple flattened format. + fn adt(properties: &[(&str, &[u8])], children: &[Vec]) -> Vec { + let mut out = Vec::new(); + out.extend((properties.len() as u32).to_le_bytes()); + out.extend((children.len() as u32).to_le_bytes()); + for (name, value) in properties { + let mut field = [0u8; 32]; + field[..name.len()].copy_from_slice(name.as_bytes()); + out.extend(field); + out.extend((value.len() as u32).to_le_bytes()); + out.extend(*value); + out.resize(out.len().next_multiple_of(4), 0); + } + for child in children { + out.extend(child); + } + out + } + + #[test] + fn a_region_lands_in_chosen_memory_map_and_other_nodes_are_untouched() { + let map = |props: &[(&str, &[u8])]| { + let mut all = vec![("name", &b"memory-map\0"[..])]; + all.extend(props); + adt(&all, &[]) + }; + let tree = |inner: Vec| { + adt( + &[("name", b"device-tree\0")], + &[ + adt(&[("name", b"cpus\0")], &[]), + adt(&[("name", b"chosen\0")], &[inner]), + adt(&[("name", b"after\0")], &[]), + ], + ) + }; + let before = tree(map(&[("RAMDisk", &[1; 16])])); + let after = add_memory_map_region(&before, "BootKC-rs", 0x1234, 0x56).unwrap(); + // The same tree, built with the property already present, is byte-identical. + let mut value = Vec::new(); + value.extend(0x1234u64.to_le_bytes()); + value.extend(0x56u64.to_le_bytes()); + assert_eq!( + after, + tree(map(&[("RAMDisk", &[1; 16]), ("BootKC-rs", &value)])) + ); + } + + #[test] + fn a_tree_without_chosen_memory_map_is_refused() { + let tree = adt(&[("name", b"device-tree\0")], &[]); + assert!(matches!( + add_memory_map_region(&tree, "BootKC-rs", 0, 0), + Err(Error::DeviceTree(_)) + )); + } +} diff --git a/src/emulator/src/system/xnu.rs b/src/emulator/src/system/xnu.rs index 4ee834a..d356eb3 100644 --- a/src/emulator/src/system/xnu.rs +++ b/src/emulator/src/system/xnu.rs @@ -70,6 +70,12 @@ pub fn is_macho(image: &[u8]) -> bool { image.len() >= 4 && u32::from_le_bytes(image[..4].try_into().unwrap()) == MH_MAGIC_64 } +/// Whether `image` is a kernel collection (`MH_FILESET`), which enters at its reset +/// trampoline rather than at `_start`. +pub fn is_fileset(image: &[u8]) -> bool { + image.len() >= 16 && u32::from_le_bytes(image[12..16].try_into().unwrap()) == MH_FILESET +} + fn bad(reason: &'static str) -> Error { Error::BadMachO(reason) } From 31584515b1329ac2cb964b3e51a85f9e13cd37ac Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Wed, 7 Oct 2026 21:53:00 -0700 Subject: [PATCH 05/13] feat(emulator): place SPTM monitor image parts in validated order - xnu::Segment keeps its segname, with Segment::name(). - monitor::image_parts names an image's contiguous parts (ro, rx, bx, rw, le, rs, virt, entry); write_part copies one; place_parts writes a whole image at caller addresses and returns its chosen/memory-map regions. - monitor::add_property inserts a property at any device-tree node path; add_memory_map_region is built on it. - The monitor's region table requires each present region to start where the previous one ended; the probe packs TXM, TrustCache, the kernelcache, the device tree and SPTM accordingly. - The monitor now validates the regions and reads /chosen/dram-base. It then zeroes a page at VA & 0x3fff_ffff_ffff, so its linear-map convention is still unresolved. --- README.md | 24 ++- src/emulator/src/system/monitor.rs | 235 ++++++++++++++++++++++++++--- src/emulator/src/system/xnu.rs | 11 ++ 3 files changed, 238 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 6a9d0e8..47e077f 100644 --- a/README.md +++ b/README.md @@ -102,14 +102,22 @@ The EL2 regime is modelled: `CurrentEl` reports EL2, exceptions and `ERET` targe firmware does: the image placed in DRAM at its physical address, the MMU off, EL2, and `x0` a boot structure (DRAM's virtual and physical base and size, the device tree, a scratch buffer). The monitor builds its own translation tables, turns the MMU on, and -reads `chosen/memory-map` for the image regions iBoot would have recorded. Its strings -(`sptm_fixup`, `unsupported auth_rebase key`) and a loop over the kernel's range suggest -it applies the kernelcache's chained fixups itself; that is inferred, not yet confirmed. -It runs about four million instructions on the real kernelcache before it panics with -`init_get_image_region: error -1 looking up image region 'TXM-ro'`, written to its scratch -buffer. The map holds one entry per image (`SPTM`, `TXM`, `BootKC`, `AuxKC`, `CL4`) and -part (`-ro`, `-rx`, `-rw`, `-bx`, `-le`, `-rs`, plus `-entry`, `-virt`, `-exception`); -volt supplies `BootKC-rs` and `BootKC-ro` only, and does not load TXM. GXF (`genter`, +checks what iBoot would have recorded in `chosen`: the `chosen/memory-map` regions, and +`dram-base` and `dram-size`. Its strings suggest it applies the kernelcache's chained +fixups itself (`sptm_fixup`); that is inferred, not confirmed. + +The region table is walked in a fixed order, and each present region must start where the +previous present one ended. In order: `TXM-ro`, `TXM-rx`, `TXM-bx`, `TrustCache`, +`AuxKC-ro/rx` (optional), `BootKC-rx`, `BootKC-bx`, `BootKC-ro`, `BootKC-rs`, +`CL4-rx/ro` (optional), `DeviceTree`, `SPTM-ro`, `SPTM-rx`, `SPTM-rw`. Each part is a +contiguous link span (`monitor::image_parts`), copied whole (`monitor::place_parts`). The +monitor's own image stays linear, because it runs position-independently before its MMU +is on. With that packing, validation passes and the monitor reaches `/chosen/dram-base`. + +It then stops in a data abort. It zeroes a page at `VA & 0x3fff_ffff_ffff` of a linear-map +address, so its convention for converting a linear-map address to a physical one is not +the one volt's boot structure implies (`VA = PA + x22 - x23`). Which convention it uses, and +where iBoot places DRAM relative to the image, is not yet established. GXF (`genter`, `gexit`, `GXF_*`, `VBAR_GL1`) and SPRR are not modelled; `genter` raises an undefined instruction. diff --git a/src/emulator/src/system/monitor.rs b/src/emulator/src/system/monitor.rs index 7f414ac..ee1fa82 100644 --- a/src/emulator/src/system/monitor.rs +++ b/src/emulator/src/system/monitor.rs @@ -66,6 +66,8 @@ pub enum Error { }, /// The device tree is not a well-formed Apple flattened device tree. DeviceTree(&'static str), + /// The entry point lies in no part of the image. + NoEntryPart, } impl From for Error { @@ -138,14 +140,13 @@ pub fn without_placeholder_flags(tree: &[u8]) -> Result, Error> { Ok(copy) } -/// Appends the property `name = {address: u64, size: u64}` to the device tree's -/// `chosen/memory-map` node, the way the boot firmware records a region it loaded (the -/// monitor looks up `BootKC-rs`, the kernel collection, and panics without it). -pub fn add_memory_map_region( +/// Appends a property `name = value` to the node at `path` of the device tree `tree` +/// (`path` names children of the root, e.g. `["chosen", "memory-map"]`). +pub fn add_property( tree: &[u8], + path: &[&[u8]], name: &str, - address: u64, - size: u64, + value: &[u8], ) -> Result, Error> { const TRUNCATED: Error = Error::DeviceTree("device tree is truncated"); const NAME: usize = 32; @@ -187,31 +188,44 @@ pub fn add_memory_map_region( } cursor = skip(tree, cursor)?; } - Err(Error::DeviceTree( - "device tree has no chosen/memory-map node", - )) - } - let chosen = child(tree, 0, b"chosen")?; - let map = child(tree, chosen, b"memory-map")?; - let (end, _) = properties(tree, map)?; - let mut property = [0u8; NAME + 4 + 16]; - let name = name.as_bytes(); - if name.len() >= NAME { - return Err(Error::DeviceTree("property name is too long")); - } - property[..name.len()].copy_from_slice(name); - property[NAME..NAME + 4].copy_from_slice(&16u32.to_le_bytes()); - property[NAME + 4..NAME + 12].copy_from_slice(&address.to_le_bytes()); - property[NAME + 12..].copy_from_slice(&size.to_le_bytes()); + Err(Error::DeviceTree("device tree has no such node")) + } + if name.len() >= NAME || value.len() > u32::MAX as usize { + return Err(Error::DeviceTree("property name or value is too large")); + } + let mut node = 0; + for part in path { + node = child(tree, node, part)?; + } + let (end, _) = properties(tree, node)?; + let mut property = vec![0u8; NAME + 4 + value.len()]; + property[..name.len()].copy_from_slice(name.as_bytes()); + property[NAME..NAME + 4].copy_from_slice(&(value.len() as u32).to_le_bytes()); + property[NAME + 4..].copy_from_slice(value); + property.resize(property.len().next_multiple_of(4), 0); let mut out = Vec::with_capacity(tree.len() + property.len()); out.extend_from_slice(&tree[..end]); out.extend_from_slice(&property); out.extend_from_slice(&tree[end..]); - let count = word(&out, map)? + 1; - out[map..map + 4].copy_from_slice(&count.to_le_bytes()); + let count = word(&out, node)? + 1; + out[node..node + 4].copy_from_slice(&count.to_le_bytes()); Ok(out) } +/// Appends the property `name = {address: u64, size: u64}` to the device tree's +/// `chosen/memory-map` node, the way the boot firmware records a region it loaded. +pub fn add_memory_map_region( + tree: &[u8], + name: &str, + address: u64, + size: u64, +) -> Result, Error> { + let mut value = Vec::with_capacity(16); + value.extend(address.to_le_bytes()); + value.extend(size.to_le_bytes()); + add_property(tree, &[b"chosen", b"memory-map"], name, &value) +} + /// Places the `MH_EXECUTE` monitor `image` and `device_tree`, and writes the boot structure. pub fn load( memory: &mut impl GuestMemory, @@ -293,6 +307,179 @@ pub fn load( }) } +/// The lowest nonempty segment address of `kernel`: the image's link base. +fn image_base(kernel: &xnu::Kernel) -> u64 { + kernel + .segments + .iter() + .filter(|s| s.vmsize != 0) + .map(|s| s.vmaddr) + .min() + .unwrap_or(0) +} + +/// Places a Mach-O image so that its lowest segment is at physical `phys`. `phys` must +/// match the link base modulo [`BLOCK`], so the monitor's linear map reaches the image at +/// its link address. Returns the physical entry point. +pub fn place_image(memory: &mut impl GuestMemory, image: &[u8], phys: u64) -> Result { + let kernel = xnu::parse(image).map_err(Error::Image)?; + let base = image_base(&kernel); + if (base ^ phys) & (BLOCK - 1) != 0 { + return Err(Error::Misaligned { + virtual_base: base, + physical_base: phys, + }); + } + write_segments(memory, image, &kernel, base, phys)?; + Ok(phys + (kernel.entry - base)) +} + +/// Writes each segment's file bytes, zero-filled to its size, at `phys + (vmaddr - base)`. +fn write_segments( + memory: &mut impl GuestMemory, + image: &[u8], + kernel: &xnu::Kernel, + base: u64, + phys: u64, +) -> Result<(), Error> { + for segment in &kernel.segments { + if segment.vmsize == 0 { + continue; + } + let mut bytes = vec![0u8; segment.vmsize as usize]; + let file = &image[segment.fileoff as usize..][..segment.filesize as usize]; + bytes[..file.len()].copy_from_slice(file); + memory.write(phys + (segment.vmaddr - base), &bytes)?; + } + Ok(()) +} + +/// A named span of a Mach-O image, in the monitor's naming: link addresses `lo..hi`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Part { + pub name: &'static str, + pub lo: u64, + pub hi: u64, +} + +/// The parts of a Mach-O image, as the monitor names them. `ro` covers `__TEXT`, +/// `__PRELINK_TEXT`, `__DATA_CONST` and `__LATE_CONST`; `rx` covers `__TEXT_EXEC` and +/// `__LAST`; `bx` is `__TEXT_BOOT_EXEC`; `rw` covers `__DATA` and `__BOOTDATA`; `le` is +/// `__LINKEDIT`; `rs` is `__DATA_SPTM` where present; `virt` is the whole image; `entry` +/// is the segment holding the entry point. A part is the smallest span covering its +/// segments and is copied as one piece, so the monitor maps it contiguously. +pub fn image_parts(image: &[u8]) -> Result, Error> { + let kernel = xnu::parse(image).map_err(Error::Image)?; + let base = image_base(&kernel); + let end = kernel + .segments + .iter() + .map(|s| s.vmaddr + s.vmsize) + .max() + .unwrap_or(base); + let span = |names: &[&[u8]]| -> Option<(u64, u64)> { + kernel + .segments + .iter() + .filter(|s| s.vmsize != 0 && names.contains(&s.name())) + .map(|s| (s.vmaddr, s.vmaddr + s.vmsize)) + .fold(None, |acc, (lo, hi)| match acc { + None => Some((lo, hi)), + Some((a, b)) => Some((a.min(lo), b.max(hi))), + }) + }; + let entry = kernel + .segments + .iter() + .find(|s| s.vmsize != 0 && (s.vmaddr..s.vmaddr + s.vmsize).contains(&kernel.entry)) + .map(|s| (s.vmaddr, s.vmaddr + s.vmsize)); + let spans = [ + ( + "ro", + span(&[ + b"__TEXT", + b"__PRELINK_TEXT", + b"__DATA_CONST", + b"__LATE_CONST", + ]), + ), + ("rx", span(&[b"__TEXT_EXEC", b"__LAST"])), + ("bx", span(&[b"__TEXT_BOOT_EXEC"])), + ("rw", span(&[b"__DATA", b"__BOOTDATA"])), + ("le", span(&[b"__LINKEDIT"])), + ("rs", span(&[b"__DATA_SPTM"])), + ("virt", Some((base, end))), + ("entry", entry), + ]; + Ok(spans + .into_iter() + .filter_map(|(name, range)| range.map(|(lo, hi)| Part { name, lo, hi })) + .collect()) +} + +/// Copies the link span of `part` from `image` to physical `pa`: the file bytes of each +/// segment inside it, zeros elsewhere. +pub fn write_part( + memory: &mut impl GuestMemory, + image: &[u8], + part: &Part, + pa: u64, +) -> Result<(), Error> { + let kernel = xnu::parse(image).map_err(Error::Image)?; + let mut bytes = vec![0u8; (part.hi - part.lo) as usize]; + for segment in &kernel.segments { + let lo = segment.vmaddr.max(part.lo); + let hi = (segment.vmaddr + segment.filesize).min(part.hi); + if lo >= hi { + continue; + } + let from = segment.fileoff + (lo - segment.vmaddr); + let file = &image[from as usize..][..(hi - lo) as usize]; + bytes[(lo - part.lo) as usize..(hi - part.lo) as usize].copy_from_slice(file); + } + memory.write(pa, &bytes)?; + Ok(()) +} + +/// Writes every part of `image` at the address `place` gives it, and returns the image's +/// `chosen/memory-map` regions as `(prefix-part, address, size)`. `entry` is not written: +/// it lies at the same offset within the part that holds the entry point. +pub fn place_parts( + memory: &mut impl GuestMemory, + image: &[u8], + prefix: &str, + place: impl Fn(&Part) -> u64, +) -> Result, Error> { + let parts = image_parts(image)?; + let mut regions = Vec::new(); + for part in parts.iter().filter(|p| p.name != "entry") { + let pa = place(part); + write_part(memory, image, part, pa)?; + regions.push((format!("{prefix}-{}", part.name), pa, part.hi - part.lo)); + } + if let Some(entry) = parts.iter().find(|p| p.name == "entry") { + let holder = parts + .iter() + .find(|p| !matches!(p.name, "entry" | "virt") && p.lo <= entry.lo && entry.hi <= p.hi) + .ok_or(Error::NoEntryPart)?; + let pa = place(holder) + (entry.lo - holder.lo); + regions.push((format!("{prefix}-entry"), pa, entry.hi - entry.lo)); + } + Ok(regions) +} + +/// Appends every region of `regions` to `chosen/memory-map` of `tree`, in order. +pub fn add_memory_map_regions( + tree: &[u8], + regions: &[(String, u64, u64)], +) -> Result, Error> { + let mut tree = tree.to_vec(); + for (name, address, size) in regions { + tree = add_memory_map_region(&tree, name, *address, *size)?; + } + Ok(tree) +} + /// Sets the CPU to the state the firmware leaves for the monitor: EL2, MMU off, at the /// entry's physical address, with the boot structure in x0. pub fn prepare_cpu(cpu: &mut crate::aarch64::cpu::Cpu, monitor: &Monitor) { diff --git a/src/emulator/src/system/xnu.rs b/src/emulator/src/system/xnu.rs index d356eb3..9a25c88 100644 --- a/src/emulator/src/system/xnu.rs +++ b/src/emulator/src/system/xnu.rs @@ -54,6 +54,16 @@ pub struct Segment { pub vmsize: u64, pub fileoff: u64, pub filesize: u64, + /// `segname`, NUL-padded. + pub name: [u8; 16], +} + +impl Segment { + /// The segment name without its NUL padding (`__TEXT`). + pub fn name(&self) -> &[u8] { + let end = self.name.iter().position(|&b| b == 0).unwrap_or(16); + &self.name[..end] + } } /// What the loader needs from a Mach-O kernel. @@ -123,6 +133,7 @@ pub fn parse(image: &[u8]) -> Result { vmsize: double(at + 32), fileoff: double(at + 40), filesize: double(at + 48), + name: image[at + 8..at + 24].try_into().unwrap(), }; let in_file = segment .fileoff From 5bc7917dcb6680e46d944b150239b9a7d8d5a174 Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Wed, 7 Oct 2026 21:56:50 -0700 Subject: [PATCH 06/13] docs(emulator): record SPTM PAPT range table finding The monitor maps physical ranges through a PAPT table ({VA base, PA base, pages}) filled by its DRAM registrar at 0xfffffff0270bd224. The registrar is never entered; its only reference is a function-pointer table in __DATA_CONST. With the table empty, a lookup misses and the zeroing routine clears a masked address. This corrects the earlier reading of the masked TTBR and zeroing addresses as a physical convention. --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 47e077f..80ce065 100644 --- a/README.md +++ b/README.md @@ -114,10 +114,16 @@ contiguous link span (`monitor::image_parts`), copied whole (`monitor::place_par monitor's own image stays linear, because it runs position-independently before its MMU is on. With that packing, validation passes and the monitor reaches `/chosen/dram-base`. -It then stops in a data abort. It zeroes a page at `VA & 0x3fff_ffff_ffff` of a linear-map -address, so its convention for converting a linear-map address to a physical one is not -the one volt's boot structure implies (`VA = PA + x22 - x23`). Which convention it uses, and -where iBoot places DRAM relative to the image, is not yet established. GXF (`genter`, +It then stops in a data abort. The monitor maps physical ranges through a table of +`{VA base, PA base, pages}` entries (the PAPT table, at `0xfffffff027105000 + 0x9d0`) that +its DRAM registrar (`0xfffffff0270bd224`) fills. The registrar checks each physical base +against `sptm_first_dram`/`sptm_last_dram`, so the physical addresses volt supplies are the +right kind. The registrar is never entered: its only reference is a function-pointer table +in `__DATA_CONST`, and the code that dispatches that table is not yet traced. With the table +empty, a lookup misses and the zeroing routine at `0xfffffff0270d05f4` clears a masked +address (`0x3fe820000000`). Earlier notes read the masked values (`VA & 0x0000_ffff_ffff_ffff` +in the TTBR writes) as a physical-address convention; they are artifacts of the same miss. +GXF (`genter`, `gexit`, `GXF_*`, `VBAR_GL1`) and SPRR are not modelled; `genter` raises an undefined instruction. From 6f24dbc9119f4830106a9fa4c087f491c87e2189 Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Wed, 7 Oct 2026 22:48:04 -0700 Subject: [PATCH 07/13] docs(emulator): trace the SPTM PAPT fill path The bootstrap fills the PAPT table in its own DRAM walk (0xfffffff0270d05f4, entry writes at d0804) and zero-fills pages through that table at 0xfffffff0270d0b0c. Function 0xfffffff0270bd224 is SPTM_FUNCTIONID_SLIDE_REGION (dispatch command), not the bootstrap's registrar. The abort's masked address is a lookup miss, not a register convention. --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 80ce065..a7631f3 100644 --- a/README.md +++ b/README.md @@ -115,17 +115,17 @@ monitor's own image stays linear, because it runs position-independently before is on. With that packing, validation passes and the monitor reaches `/chosen/dram-base`. It then stops in a data abort. The monitor maps physical ranges through a table of -`{VA base, PA base, pages}` entries (the PAPT table, at `0xfffffff027105000 + 0x9d0`) that -its DRAM registrar (`0xfffffff0270bd224`) fills. The registrar checks each physical base -against `sptm_first_dram`/`sptm_last_dram`, so the physical addresses volt supplies are the -right kind. The registrar is never entered: its only reference is a function-pointer table -in `__DATA_CONST`, and the code that dispatches that table is not yet traced. With the table -empty, a lookup misses and the zeroing routine at `0xfffffff0270d05f4` clears a masked -address (`0x3fe820000000`). Earlier notes read the masked values (`VA & 0x0000_ffff_ffff_ffff` -in the TTBR writes) as a physical-address convention; they are artifacts of the same miss. -GXF (`genter`, -`gexit`, `GXF_*`, `VBAR_GL1`) and SPRR are not modelled; `genter` raises an undefined -instruction. +`{VA base, PA base, pages}` entries (the PAPT table, at `0xfffffff027105000 + 0x9d0`) that its registrar +(`0xfffffff0270bd224`) fills. That function is `SPTM_FUNCTIONID_SLIDE_REGION` +(found in a dispatch table at `0xfffffff027021dd0`): it registers one physical range into +the PAPT table for callers that send the command; it checks each physical base against +`sptm_first_dram`/`sptm_last_dram`. The bootstrap does not call it. The bootstrap fills +the table in its own DRAM walk (`0xfffffff0270d05f4`, the entry writes at `0xfffffff0270d0804`), +then zeroes authored pages at `0xfffffff0270d0b0c` through the same table. The abort is a +lookup miss in that walk (`0x3fe820000000`), after which earlier notes about a +`VA & 0x0000_ffff_ffff_ffff` physical-address convention are artifacts of the miss, not a +register convention. GXF (`genter`, `gexit`, `GXF_*`, `VBAR_GL1`) and SPRR are not +modelled; `genter` raises an undefined instruction. Two debugging hooks make this tractable. `Machine::set_step_trace` runs one instruction per block and reports each step with the system registers it changed, exception entries From ae0d8329440e065ec898b20a264f07f9d5c5a080 Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Thu, 8 Oct 2026 00:07:08 -0700 Subject: [PATCH 08/13] feat(emulator): add bios option --- src/emulator/cli/src/boot.rs | 99 +++++++++++++++-- src/emulator/src/aarch64/compile.rs | 26 ++++- src/emulator/src/aarch64/cpu.rs | 6 ++ src/emulator/src/aarch64/decode.rs | 19 ++++ src/emulator/src/aarch64/translate.rs | 53 +++++++-- src/emulator/src/memory.rs | 75 +++++++++++-- src/emulator/src/system/boot.rs | 29 +++++ src/emulator/src/system/mod.rs | 149 +++++++++++++++++++++++--- 8 files changed, 409 insertions(+), 47 deletions(-) diff --git a/src/emulator/cli/src/boot.rs b/src/emulator/cli/src/boot.rs index 9ccd3cd..84bf399 100644 --- a/src/emulator/cli/src/boot.rs +++ b/src/emulator/cli/src/boot.rs @@ -1,4 +1,5 @@ -//! Raw Linux arm64 Image, or XNU Mach-O kernel, boot through Volt's full-system JIT. +//! Raw Linux arm64 Image, XNU Mach-O kernel, or `-bios` firmware image, booted through Volt's +//! full-system JIT. use std::{ io::Write, @@ -8,8 +9,8 @@ use std::{ }; use volt_emulator::devices::BlockBackend; use volt_emulator::system::{ - Board, DEFAULT_CMDLINE, DEFAULT_XNU_CMDLINE, IdleMode, RAM_SIZE, SystemConfig, XNU_RAM_SIZE, - m1, smp::SmpConfig, + Board, DEFAULT_CMDLINE, DEFAULT_XNU_CMDLINE, FIRMWARE_RAM_SIZE, IdleMode, RAM_SIZE, + SystemConfig, XNU_RAM_SIZE, m1, smp::SmpConfig, }; use winnow_args::{Args, ValueEnum}; @@ -36,12 +37,12 @@ impl From for IdleMode { #[derive(Args, Debug)] #[arg( name = "volt-boot", - about = "AArch64 Linux or XNU boot with PL011, GICv2/GICv3, VirtIO-Blk, or the Apple M1 SoC" + about = "AArch64 Linux, XNU or firmware boot with PL011, GICv2/GICv3, VirtIO-Blk, or the Apple M1 SoC" )] struct Cli { - /// Path to a raw Linux AArch64 Image, or an XNU kernel (Mach-O). + /// Path to a raw Linux AArch64 Image, or an XNU kernel (Mach-O). Omit with --bios. #[arg(positional)] - image: PathBuf, + image: Option, /// Timeout in seconds (default: 20). #[arg(positional)] @@ -64,6 +65,15 @@ struct Cli { #[arg(long)] dtb: Option, + /// Firmware image to run from flash at address 0, as QEMU's `-bios` does (e.g. an EDK2 + /// `QEMU_EFI.fd` or tinted-boot's `tinted-boot-aarch64.bin`). Boots no kernel. + #[arg(long)] + bios: Option, + + /// Timeout in seconds, for any boot (default: 20). Alternative to the seconds positional. + #[arg(long)] + timeout: Option, + /// Guest RAM size (e.g. 128M, 256M, 512M, 1G). #[arg(short, long, alias = "ram")] memory: Option, @@ -161,6 +171,12 @@ fn main() -> ExitCode { } else if s == "-smp" { args.push("--smp".into()); continue; + } else if s == "-bios" { + args.push("--bios".into()); + continue; + } else if let Some(rest) = s.strip_prefix("-bios=") { + args.push(format!("--bios={rest}").into()); + continue; } } args.push(arg); @@ -196,8 +212,15 @@ fn run(cli: Cli) -> Result<(), String> { IdleMode::Paced }; - let seconds = cli.seconds.unwrap_or(20); - let image = std::fs::read(&cli.image).map_err(|e| format!("{}: {e}", cli.image.display()))?; + let seconds = cli.timeout.or(cli.seconds).unwrap_or(20); + if let Some(bios) = &cli.bios { + return run_firmware(&cli, bios, idle_mode, seconds); + } + let image_path = cli + .image + .as_ref() + .ok_or("no kernel image given: pass a Linux Image or XNU kernel, or --bios FIRMWARE")?; + let image = std::fs::read(image_path).map_err(|e| format!("{}: {e}", image_path.display()))?; let board = match &cli.dtb { Some(path) => Board::AppleM1 { device_tree: std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?, @@ -288,6 +311,54 @@ fn run(cli: Cli) -> Result<(), String> { Ok(()) } +/// Boots a firmware image as QEMU's `-bios` does: no kernel, the image run from flash at 0. +fn run_firmware(cli: &Cli, bios: &Path, idle_mode: IdleMode, seconds: u64) -> Result<(), String> { + if cli.image.is_some() { + return Err( + "--bios boots firmware without a kernel; drop the positional arguments and use --timeout" + .into(), + ); + } + if cli.dtb.is_some() { + return Err("--bios and --dtb select different machines".into()); + } + if cli.cpus > 1 { + return Err("firmware boot has one vCPU for now; pass --smp 1".into()); + } + let firmware = std::fs::read(bios).map_err(|e| format!("{}: {e}", bios.display()))?; + let ram_size = match cli.memory.as_deref() { + Some(s) => parse_memory_size(s)?, + None => FIRMWARE_RAM_SIZE, + }; + let initrd = match &cli.initrd { + Some(path) => Some(load_initrd(path)?), + None => None, + }; + let disk = match &cli.disk { + Some(path) => Some(FileBackend::open(path)?), + None => None, + }; + let config = SystemConfig { + timeout: Duration::from_secs(seconds), + cmdline: String::new(), + idle_mode, + ram_size, + initrd, + disk, + board: Board::Virt, + }; + let mut output = std::io::BufWriter::new(std::io::stdout()); + let report = volt_emulator::system::boot_firmware(&firmware, config, move |byte| { + let _ = output.write_all(&[byte]); + if byte == b'\n' { + let _ = output.flush(); + } + }) + .map_err(|e| format!("{e}"))?; + println!("\n{report}"); + Ok(()) +} + fn load_initrd(path: &Path) -> Result, String> { let bytes = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?; if bytes.len() >= 4 && bytes[..4] == [0x28, 0xb5, 0x2f, 0xfd] { @@ -368,6 +439,18 @@ mod tests { assert_eq!(cli.initrd, Some(PathBuf::from("initramfs.cpio"))); } + #[test] + fn test_winnow_args_bios_needs_no_kernel() { + let cli = Cli::parse_from(["--bios", "tinted-boot-aarch64.bin", "--timeout", "7"]).unwrap(); + assert_eq!(cli.bios, Some(PathBuf::from("tinted-boot-aarch64.bin"))); + assert!(cli.image.is_none()); + assert_eq!(cli.timeout, Some(7)); + + let cli = Cli::parse_from(["Image"]).unwrap(); + assert!(cli.bios.is_none()); + assert_eq!(cli.image, Some(PathBuf::from("Image"))); + } + #[test] fn test_zstd_decompression() { let data = b"sample cpio content for zstd test"; diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index 75691bc..757e75c 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -2443,7 +2443,10 @@ fn system_offset(r: SystemRegister) -> Option { IccBpr0El1 => offset_of!(CpuSystem, icc_bpr0_el1), IccCtlrEl1 => offset_of!(CpuSystem, icc_ctlr_el1), IccIgrpen0El1 => offset_of!(CpuSystem, icc_igrpen0_el1), + IccIgrpen1El1 => offset_of!(CpuSystem, icc_igrpen1_el1), + IccBpr1El1 => offset_of!(CpuSystem, icc_bpr1_el1), CntpCtlEl0 => offset_of!(CpuSystem, cntp_ctl_el0), + CntpCvalEl0 => offset_of!(CpuSystem, cntp_cval_el0), ActlrEl1 => offset_of!(CpuSystem, actlr_el1), VbarEl1 => offset_of!(CpuSystem, vbar_el1), SctlrEl2 => offset_of!(CpuSystem, sctlr_el2), @@ -2724,11 +2727,17 @@ fn lower_system(l: &Lower, a: System) { l.sel(l.u64, is(0), l.k(l.u64, CCSIDR_L1D), l1_instruction), ); } - CntvTvalEl0 => { + CntvTvalEl0 | CntpTvalEl0 => { // The timer value is the signed 32-bit distance to the compare value: // reads give the low 32 bits of `CVAL - count`, writes `CVAL = count + sext(value)`. + // The physical timer shares the virtual count, as CNTVOFF_EL2 is zero without EL2. let counter = l.load(l.u64, at + offset_of!(CpuSystem, cntvct_el0)); - let compare = at + offset_of!(CpuSystem, cntv_cval_el0); + let compare = at + + if a.register == CntpTvalEl0 { + offset_of!(CpuSystem, cntp_cval_el0) + } else { + offset_of!(CpuSystem, cntv_cval_el0) + }; if a.read { let distance = l.bin(l.u64, B::Sub, l.load(l.u64, compare), counter); l.put(a.rt, l.imm(l.u64, B::BitAnd, distance, 0xffff_ffff)); @@ -2737,6 +2746,19 @@ fn lower_system(l: &Lower, a: System) { l.store(compare, l.bin(l.u64, B::Add, counter, delta)); } } + CntpctEl0 => { + // Writes are ignored: the counter is not writable from software. + if a.read { + l.put(a.rt, l.load(l.u64, at + offset_of!(CpuSystem, cntvct_el0))); + } + } + IccIar1El1 => { + // No group 1 interrupt is delivered by this model, so acknowledge reads spurious. + if a.read { + l.put(a.rt, l.k(l.u64, 1023)); + } + } + IccEoir1El1 => {} Daif => { let p = at + offset_of!(CpuSystem, daif); if a.read { diff --git a/src/emulator/src/aarch64/cpu.rs b/src/emulator/src/aarch64/cpu.rs index 01af196..9fd37f1 100644 --- a/src/emulator/src/aarch64/cpu.rs +++ b/src/emulator/src/aarch64/cpu.rs @@ -216,7 +216,10 @@ pub struct System { pub icc_bpr0_el1: u64, pub icc_ctlr_el1: u64, pub icc_igrpen0_el1: u64, + pub icc_igrpen1_el1: u64, + pub icc_bpr1_el1: u64, pub cntp_ctl_el0: u64, + pub cntp_cval_el0: u64, pub actlr_el1: u64, /// EL2 banks (SPTM runs at EL2, GL2). `SP_EL2` is `sp_el[2]`. pub sctlr_el2: u64, @@ -298,7 +301,10 @@ impl Default for System { // PRIbits 7: eight priority bits. icc_ctlr_el1: 7 << 8, icc_igrpen0_el1: 0, + icc_igrpen1_el1: 0, + icc_bpr1_el1: 0, cntp_ctl_el0: 0, + cntp_cval_el0: 0, actlr_el1: 0, apple: [0; crate::aarch64::decode::APPLE_SLOTS], midr_el1: crate::aarch64::identification::DEFAULT_MIDR_EL1, diff --git a/src/emulator/src/aarch64/decode.rs b/src/emulator/src/aarch64/decode.rs index e733cef..8611d7b 100644 --- a/src/emulator/src/aarch64/decode.rs +++ b/src/emulator/src/aarch64/decode.rs @@ -609,8 +609,20 @@ pub enum SystemRegister { IccBpr0El1, IccCtlrEl1, IccIgrpen0El1, + /// Group 1 enable and binary point: the group the firmware and Linux use for IRQs. + IccIgrpen1El1, + IccBpr1El1, + /// Group 1 acknowledge and end of interrupt. No group 1 interrupt is delivered by this + /// model, so an acknowledge reads spurious and an end of interrupt has nothing to end. + IccIar1El1, + IccEoir1El1, /// Stored, but the physical timer never fires. CntpCtlEl0, + /// The physical counter (the virtual count, as CNTVOFF_EL2 is zero without EL2). + CntpctEl0, + /// Physical timer compare value and its 32-bit view, sharing the counter. It never fires. + CntpTvalEl0, + CntpCvalEl0, ActlrEl1, Nzcv, RazWi, @@ -1304,7 +1316,14 @@ fn system_register( 0x0c83 => IccBpr0El1, 0x0cc4 => IccCtlrEl1, 0x0cc6 => IccIgrpen0El1, + 0x0cc7 => IccIgrpen1El1, + 0x0cc3 => IccBpr1El1, 0x3e21 => CntpCtlEl0, + 0x3e01 => CntpctEl0, + 0x3e20 => CntpTvalEl0, + 0x3e22 => CntpCvalEl0, + 0x0cc0 => IccIar1El1, + 0x0cc1 => IccEoir1El1, 0x0101 => ActlrEl1, 0x0000 => MidrEl1, 0x0005 => MpidrEl1, diff --git a/src/emulator/src/aarch64/translate.rs b/src/emulator/src/aarch64/translate.rs index 95ad5ea..81ec534 100644 --- a/src/emulator/src/aarch64/translate.rs +++ b/src/emulator/src/aarch64/translate.rs @@ -169,7 +169,9 @@ pub fn translate( } let regime = Regime::of(cpu); let t1sz = ((regime.tcr >> 16) & 63) as u32; - if regime.split && t1sz == 0 { + // TTBR1's region size only means something while EPD1 leaves its walk enabled. + let upper_walk = regime.split && regime.tcr & (1 << 23) == 0; + if upper_walk && t1sz == 0 { return Err(TranslateError::MalformedTables); } if ((virtual_address >> 55) & 1 != 0 && virtual_address >> 48 != 0xffff) @@ -181,7 +183,7 @@ pub fn translate( let upper_bits = 64 - t1sz; let upper = if virtual_address >> lower_bits == 0 { false - } else if regime.split && virtual_address >> upper_bits == u64::MAX >> upper_bits { + } else if upper_walk && virtual_address >> upper_bits == u64::MAX >> upper_bits { true } else { return Err(TranslateError::TranslationFault); @@ -274,9 +276,6 @@ fn walk( let descriptor = u64::from_le_bytes(word); let kind = descriptor & 3; if kind == 3 && level != 0 { - if descriptor & 0x3f0 != 0 { - return Err(TranslateError::MalformedTables); - } no_user |= descriptor & (1 << 61) != 0; read_only |= descriptor & (1 << 62) != 0; pxn_table |= descriptor & (1 << 59) != 0; @@ -505,7 +504,7 @@ mod tests { assert_eq!(memory.reads.get(), reads, "same page must not walk again"); } #[test] - fn invalid_descriptors_and_malformed_tables_fail_closed() { + fn invalid_descriptors_fail_closed() { let cpu = cpu(); let mut memory = Memory::new(); let mut tlb = Tlb::new(); @@ -521,11 +520,22 @@ mod tests { ); // A fault is not cached: the repeated access walks the tables again. assert!(memory.reads.get() > count); - tlb.flush(); - memory.put(0, 0x1013); + } + #[test] + fn table_descriptor_bits_eleven_to_two_are_ignored() { + // Table descriptors ignore bits 11:2 (ARM ARM D8.3.1). Set them on every table on the + // walk for address 0: the page at 0x4000_0000 must still resolve. + let cpu = cpu(); + let mut memory = Memory::new(); + let mut tlb = Tlb::new(); + memory.chain(0, 0, 12, 4, 0, 0x4000_0403); + for table in [0usize, 0x1000, 0x2000] { + let descriptor = memory.get(table); + memory.put(table as u64, descriptor | 0x3f0); + } assert_eq!( translate(&mut tlb, &cpu, &mut memory, 0, Access::Read), - Err(TranslateError::MalformedTables) + Ok(0x4000_0000) ); } #[test] @@ -770,4 +780,29 @@ mod tests { Err(TranslateError::TranslationFault) ); } + #[test] + fn epd1_disables_the_upper_walk_so_its_t1sz_is_never_checked() { + // The tinted-boot firmware's TCR_EL1: T0SZ 25 (a 39-bit lower range), EPD1 set, + // and T1SZ 0, which is only meaningful while the TTBR1 walk is enabled. + let mut cpu = cpu(); + cpu.system.tcr_el1 = 25 | (1 << 23) | (2 << 30) | (4 << 32); + let mut memory = Memory::new(); + let mut tlb = Tlb::new(); + // A 1 GiB block at 0x4000_0000 in the TTBR0 root table, which sits at 0. + memory.chain(0, 0x4000_0000, 12, 3, 2, 0x4000_0401); + assert_eq!( + translate(&mut tlb, &cpu, &mut memory, 0x4000_0000, Access::Execute), + Ok(0x4000_0000) + ); + assert_eq!( + translate( + &mut tlb, + &cpu, + &mut memory, + 0xffff_8000_0000_0000, + Access::Read + ), + Err(TranslateError::TranslationFault) + ); + } } diff --git a/src/emulator/src/memory.rs b/src/emulator/src/memory.rs index f98d324..e5e6d06 100644 --- a/src/emulator/src/memory.rs +++ b/src/emulator/src/memory.rs @@ -140,9 +140,12 @@ impl CodeTracker { } } -/// Host-visible RAM or a physical region without a host mapping. +/// Host-visible RAM, read-only storage such as flash, or a physical region without +/// a host mapping. Read-only storage serves reads and refuses writes; it has no +/// host pointer, so generated code's stores cannot bypass the refusal. pub enum Backing { Shared(Vec), + ReadOnly(Vec), Private, } @@ -165,6 +168,20 @@ impl Region { backing: Backing::Shared(bytes), }) } + + /// Read-only storage at `gpa`, such as a firmware image in flash. + pub fn rom(gpa: u64, bytes: Vec) -> Result { + let len = bytes.len() as u64; + gpa.checked_add(len).ok_or(MemoryError::AccessFault { + address: gpa, + length: bytes.len(), + })?; + Ok(Self { + gpa, + len, + backing: Backing::ReadOnly(bytes), + }) + } } /// Owns RAM storage. Every access must fit one physical region, as in Mirage. @@ -184,7 +201,7 @@ impl PhysicalMemory { address: region.gpa, length: usize::MAX, })?; - if let Backing::Shared(bytes) = ®ion.backing { + if let Backing::Shared(bytes) | Backing::ReadOnly(bytes) = ®ion.backing { if bytes.len() as u64 != region.len { return Err(MemoryError::AccessFault { address: region.gpa, @@ -228,7 +245,7 @@ impl PhysicalMemory { pub fn slice(&self, address: u64, length: usize) -> Result<&[u8], MemoryError> { let (index, offset) = self.find(address, length)?; match &self.regions[index].backing { - Backing::Shared(bytes) => bytes + Backing::Shared(bytes) | Backing::ReadOnly(bytes) => bytes .get(offset..offset + length) .ok_or(MemoryError::AccessFault { address, length }), Backing::Private => Err(MemoryError::PrivateMemory), @@ -237,14 +254,17 @@ impl PhysicalMemory { pub fn slice_mut(&mut self, address: u64, length: usize) -> Result<&mut [u8], MemoryError> { let (index, offset) = self.find(address, length)?; - // The caller writes through the returned slice, so it counts as a write - // already. The borrow (plus the RwLock for shared memory) keeps readers - // from observing the bumped version with the old bytes. - self.tracker.note_write(address, length); match &mut self.regions[index].backing { - Backing::Shared(bytes) => bytes - .get_mut(offset..offset + length) - .ok_or(MemoryError::AccessFault { address, length }), + Backing::Shared(bytes) => { + // The caller writes through the returned slice, so it counts as a write + // already. The borrow (plus the RwLock for shared memory) keeps readers + // from observing the bumped version with the old bytes. + self.tracker.note_write(address, length); + bytes + .get_mut(offset..offset + length) + .ok_or(MemoryError::AccessFault { address, length }) + } + Backing::ReadOnly(_) => Err(MemoryError::AccessFault { address, length }), Backing::Private => Err(MemoryError::PrivateMemory), } } @@ -302,7 +322,7 @@ impl SharedMemory { .iter() .filter_map(|region| match ®ion.backing { Backing::Shared(bytes) => Some((region.gpa, region.len, bytes.as_ptr().cast_mut())), - Backing::Private => None, + Backing::ReadOnly(_) | Backing::Private => None, }) .collect(); Self { @@ -424,6 +444,39 @@ mod tests { assert_eq!(memory.write(0x8000, b"x"), Err(MemoryError::PrivateMemory)); } + #[test] + fn read_only_regions_serve_reads_and_refuse_writes() { + let mut memory = + PhysicalMemory::new(vec![Region::rom(0, b"flash!".to_vec()).unwrap()]).unwrap(); + let mut out = [0u8; 5]; + memory.read(1, &mut out).unwrap(); + assert_eq!(&out, b"lash!"); + assert_eq!( + memory.write(0, b"x"), + Err(MemoryError::AccessFault { + address: 0, + length: 1 + }) + ); + assert_eq!(memory.slice(0, 6).unwrap(), b"flash!"); + } + + #[cfg(feature = "system")] + #[test] + fn read_only_regions_have_no_host_pointer_so_stores_cannot_bypass_them() { + let mut shared = SharedMemory::new( + PhysicalMemory::new(vec![Region::rom(0, vec![0xd5; 0x1000]).unwrap()]).unwrap(), + ); + assert_eq!(shared.host_page(0), None); + let mut out = [0u8; 4]; + shared.read(0, &mut out).unwrap(); + assert_eq!(out, [0xd5; 4]); + assert!(matches!( + shared.write(0, &[0]), + Err(MemoryError::AccessFault { .. }) + )); + } + #[cfg(feature = "system")] #[test] fn shared_memory_ram_access_is_lock_free_but_still_tracked() { diff --git a/src/emulator/src/system/boot.rs b/src/emulator/src/system/boot.rs index bb6d6b6..64faa61 100644 --- a/src/emulator/src/system/boot.rs +++ b/src/emulator/src/system/boot.rs @@ -195,6 +195,35 @@ pub fn place_with_tree( }) } +/// Where a firmware image is mapped: flash at address 0, where the core resets. +pub const FIRMWARE_BASE: u64 = 0; + +/// Place a device tree at the base of RAM and enter the firmware at [`FIRMWARE_BASE`] with +/// `x0` zero. That is what QEMU's `-bios` does: its reset path sets only the PC, and the +/// tree goes to RAM base for a firmware that looks there. Without a tree in `x0` the +/// tinted-boot firmware assumes its built-in platform description, which places RAM up to +/// 4 GiB; passing the tree there would have it reserve memory it already owns. +pub fn prepare_firmware( + memory: &mut impl GuestMemory, + ram_base: u64, + ram_size: u64, +) -> Result { + let tree = super::fdt::build(ram_base, ram_size, ""); + let end = ram_base.checked_add(ram_size).ok_or(Error::NoRoom)?; + if ram_base + .checked_add(tree.len() as u64) + .is_none_or(|tree_end| tree_end > end) + { + return Err(Error::NoRoom); + } + memory.write(ram_base, &tree).map_err(Error::Memory)?; + Ok(Layout { + entry: FIRMWARE_BASE, + boot_info: 0, + initrd: None, + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index 24378b2..a4fc020 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -34,6 +34,10 @@ pub const DEFAULT_CMDLINE: &str = "console=ttyAMA0 earlycon=pl011,0x9000000 noka pub const DEFAULT_XNU_CMDLINE: &str = "-v serial=3 debug=0x14e keepsyms=1 serial-device-name=uart0"; /// XNU sizes its zones from RAM; 128 MiB leaves it little to work with. pub const XNU_RAM_SIZE: usize = 1 << 30; +/// RAM a firmware boot gets by default: 4 GiB, which is the RAM the firmware's built-in +/// platform description assumes when no device tree is passed (QEMU's run script uses `-m 4G`). +/// The region is zero-filled lazily by the host, so it costs what the guest touches. +pub const FIRMWARE_RAM_SIZE: usize = 4 << 30; #[derive(Debug)] pub enum Error { @@ -419,6 +423,33 @@ pub fn boot_with_options( pub fn boot_system( image: &[u8], config: SystemConfig, + sink: impl FnMut(u8) + Send + 'static, +) -> Result { + boot_machine(Payload::Kernel(image), config, sink) +} + +/// Boot a firmware image the way QEMU's `-bios` does: the image is mapped read-only at +/// [`boot::FIRMWARE_BASE`], the core resets there, and no kernel is loaded. The machine +/// is the `virt` board with a GICv3, as tinted-boot is run. As under QEMU's `-bios`, `x0` +/// is zero and a device tree sits at RAM base. Firmware boots take no initrd or disk yet. +pub fn boot_firmware( + firmware: &[u8], + config: SystemConfig, + sink: impl FnMut(u8) + Send + 'static, +) -> Result { + boot_machine(Payload::Firmware(firmware), config, sink) +} + +/// What a boot starts: a kernel loaded into RAM, or firmware run from flash. +#[derive(Clone, Copy)] +enum Payload<'a> { + Kernel(&'a [u8]), + Firmware(&'a [u8]), +} + +fn boot_machine( + payload: Payload<'_>, + config: SystemConfig, mut sink: impl FnMut(u8) + Send + 'static, ) -> Result { let ram_size = if config.ram_size > 0 { @@ -430,21 +461,39 @@ pub fn boot_system( let idle_mode = config.idle_mode; let apple = matches!(config.board, Board::AppleM1 { .. }); let ram_base = if apple { m1::RAM_BASE } else { RAM_BASE }; - let region = Region::ram(ram_base, vec![0; ram_size]).map_err(Error::Memory)?; - let memory = PhysicalMemory::new(vec![region]).map_err(Error::Memory)?; + let is_xnu = matches!(payload, Payload::Kernel(image) if xnu::is_macho(image)); + let mut regions = vec![Region::ram(ram_base, vec![0; ram_size]).map_err(Error::Memory)?]; + if let Payload::Firmware(flash) = payload { + regions.push(Region::rom(boot::FIRMWARE_BASE, flash.to_vec()).map_err(Error::Memory)?); + } + let memory = PhysicalMemory::new(regions).map_err(Error::Memory)?; let mut shared_mem = SharedMemory::new(memory); let has_virtio = config.disk.is_some(); - let is_xnu = xnu::is_macho(image); - let layout = match config.board { - Board::AppleM1 { device_tree } if is_xnu => xnu::prepare_boot_with_tree( - &mut shared_mem, - image, - &config.cmdline, - &device_tree, - ram_base, - ram_size as u64, - ), - Board::AppleM1 { device_tree } => { + let layout = match (payload, config.board) { + (Payload::Firmware(_), Board::AppleM1 { .. }) => { + return Err(Error::Boot(boot::Error::Unsupported( + "the Apple M1 SoC boots no firmware image yet", + ))); + } + (Payload::Firmware(_), Board::Virt) => { + if config.initrd.is_some() || has_virtio { + return Err(Error::Boot(boot::Error::Unsupported( + "firmware boot has no initrd or virtio-blk yet; the firmware reads its disk over PCI", + ))); + } + boot::prepare_firmware(&mut shared_mem, ram_base, ram_size as u64) + } + (Payload::Kernel(image), Board::AppleM1 { device_tree }) if is_xnu => { + xnu::prepare_boot_with_tree( + &mut shared_mem, + image, + &config.cmdline, + &device_tree, + ram_base, + ram_size as u64, + ) + } + (Payload::Kernel(image), Board::AppleM1 { device_tree }) => { if has_virtio || config.initrd.is_some() { return Err(Error::Boot(boot::Error::Unsupported( "the Apple M1 SoC has no initrd or virtio disk support yet", @@ -458,7 +507,7 @@ pub fn boot_system( ram_size as u64, ) } - Board::Virt if is_xnu => { + (Payload::Kernel(image), Board::Virt) if is_xnu => { if config.initrd.is_some() || has_virtio { return Err(Error::Boot(boot::Error::Unsupported( "XNU boot has no initrd or virtio disk support", @@ -472,7 +521,7 @@ pub fn boot_system( ram_size as u64, ) } - Board::Virt => boot::prepare_boot( + (Payload::Kernel(image), Board::Virt) => boot::prepare_boot( &mut shared_mem, image, config.initrd.as_deref(), @@ -497,7 +546,7 @@ pub fn boot_system( } else { let serial = Pl011::new(emit).with_line(move |level| serial_line.store(level, Ordering::Release)); - let gic = if is_xnu { + let gic = if is_xnu || matches!(payload, Payload::Firmware(_)) { Intc::V3(Gicv3::default()) } else { Intc::V2(Gicv2::default()) @@ -514,10 +563,15 @@ pub fn boot_system( if apple { m1::identify(&mut machine.cpu); } + if matches!(payload, Payload::Firmware(_)) { + // The firmware drives a GICv3 through `ICC_*` system registers, and checks the GIC + // field of ID_AA64PFR0_EL1 before it uses them, as QEMU reports for gic-version=3. + machine.cpu.system.id_aa64pfr0_el1 |= 1 << 24; + } machine.cpu.pc = layout.entry; machine.cpu.x[0] = layout.boot_info; machine.cpu.x[1..4].fill(0); - if is_xnu && xnu::is_fileset(image) { + if matches!(payload, Payload::Kernel(image) if is_xnu && xnu::is_fileset(image)) { // A kernel collection enters at its reset trampoline: x0 is the reset type (0 for // a cold boot) and x1 the boot arguments. machine.cpu.x[0] = 0; @@ -929,4 +983,65 @@ mod tests { assert_eq!(report.reason, StopReason::Shutdown); assert_eq!(report.console, b"C"); } + + /// A raw little-endian instruction stream, as a firmware image is laid out in flash. + fn flash_image(instructions: &[u32]) -> Vec { + instructions.iter().flat_map(|i| i.to_le_bytes()).collect() + } + + #[test] + fn firmware_runs_from_flash_with_x0_zero_as_under_qemu_bios() { + // mov x19, x0 keeps the entry value of x0; then 'F' to the PL011, then PSCI off. + let flash = flash_image(&[ + 0xaa0003f3, // mov x19, x0 + 0xd2a12001, // movz x1, #0x900, lsl #16 (PL011) + 0x528008c2, // movz w2, #70 ('F') + 0x39000022, // strb w2, [x1] + 0xd2b08000, // movz x0, #0x8400, lsl #16 + 0xf2800100, // movk x0, #8 (PSCI_SYSTEM_OFF) + 0xd4000002, // hvc #0 + ]); + let report = boot_firmware( + &flash, + SystemConfig::> { + timeout: Duration::from_secs(5), + cmdline: String::new(), + idle_mode: IdleMode::Paced, + ram_size: 256 << 20, + initrd: None, + disk: None, + board: Board::Virt, + }, + |_| {}, + ) + .unwrap(); + assert_eq!(report.reason, StopReason::Shutdown, "{report}"); + assert_eq!(report.console, b"F"); + assert_eq!(report.layout.entry, boot::FIRMWARE_BASE); + assert_eq!(report.layout.boot_info, 0); + assert_eq!(report.cpu.x[19], 0); + assert_eq!(report.unmapped_accesses, 0); + } + + #[test] + fn firmware_boot_refuses_a_disk_it_cannot_reach() { + let flash = flash_image(&[0xd4000002]); + let result = boot_firmware( + &flash, + SystemConfig { + timeout: Duration::from_secs(5), + cmdline: String::new(), + idle_mode: IdleMode::Paced, + ram_size: 256 << 20, + initrd: None, + disk: Some(vec![0u8; 512]), + board: Board::Virt, + }, + |_| {}, + ); + assert!(matches!( + result, + Err(Error::Boot(boot::Error::Unsupported(_))) + )); + } } From 2783fdb85d55ec5e7f19d6853767fedbc024992c Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Thu, 8 Oct 2026 00:28:02 -0700 Subject: [PATCH 09/13] feat(emulator): add virtio pci --- src/emulator/cli/src/boot.rs | 3 +- src/emulator/src/devices/gicv3.rs | 16 +- src/emulator/src/devices/mod.rs | 4 + src/emulator/src/devices/pci.rs | 427 ++++++++++++++++++++ src/emulator/src/devices/virtio_blk.rs | 418 ++++++++++--------- src/emulator/src/devices/virtio_pci.rs | 535 +++++++++++++++++++++++++ src/emulator/src/system/machine.rs | 19 + src/emulator/src/system/mod.rs | 133 ++++-- 8 files changed, 1329 insertions(+), 226 deletions(-) create mode 100644 src/emulator/src/devices/pci.rs create mode 100644 src/emulator/src/devices/virtio_pci.rs diff --git a/src/emulator/cli/src/boot.rs b/src/emulator/cli/src/boot.rs index 84bf399..683fcbc 100644 --- a/src/emulator/cli/src/boot.rs +++ b/src/emulator/cli/src/boot.rs @@ -52,7 +52,8 @@ struct Cli { #[arg(positional)] cmdline: Option, - /// Disk image for virtio-blk rootfs (e.g. rootfs.ext4, disk.img). + /// Disk image (e.g. rootfs.ext4, disk.img). A kernel boot gets it as virtio-blk on MMIO; a + /// `--bios` boot as virtio-blk on PCI, at 00:01.0. #[arg(short, long, alias = "drive", alias = "rootfs")] disk: Option, diff --git a/src/emulator/src/devices/gicv3.rs b/src/emulator/src/devices/gicv3.rs index 8611ca6..c1c521b 100644 --- a/src/emulator/src/devices/gicv3.rs +++ b/src/emulator/src/devices/gicv3.rs @@ -50,6 +50,17 @@ pub struct Gicv3 { enabled: u32, pending: u32, priority: [u8; 32], + /// `ITLinesNumber` as the distributor reports it. Zero is a board with 32 INTIDs and no + /// shared interrupts; firmware that registers interrupts sized by `arm-gic` needs more. + it_lines: u8, +} + +impl Gicv3 { + /// A distributor with `it_lines` interrupt lines (see the field). + pub fn with_it_lines(mut self, it_lines: u8) -> Self { + self.it_lines = it_lines; + self + } } impl Default for Gicv3 { @@ -62,6 +73,7 @@ impl Default for Gicv3 { enabled: 0, pending: 0, priority: [0; 32], + it_lines: 0, } } } @@ -82,8 +94,8 @@ impl Gicv3 { // All of these are 32-bit registers. let register = match offset & !3 { GICD_CTLR => u64::from(self.distributor_ctlr), - // `ITLinesNumber` 0 (32 INTIDs), one PE, 16 ID bits. - GICD_TYPER => 15 << 19, + // `ITLinesNumber` (`it_lines`) gives 32 * (it_lines + 1) INTIDs, one PE, 16 ID bits. + GICD_TYPER => (15 << 19) | u64::from(self.it_lines), PIDR2 => PIDR2_GICV3, _ => 0, }; diff --git a/src/emulator/src/devices/mod.rs b/src/emulator/src/devices/mod.rs index fa62444..60b57e3 100644 --- a/src/emulator/src/devices/mod.rs +++ b/src/emulator/src/devices/mod.rs @@ -2,13 +2,17 @@ pub mod aic; pub mod bus; pub mod gicv2; pub mod gicv3; +pub mod pci; pub mod pl011; pub mod pmgr; pub mod s5l_uart; pub mod virtio_blk; +pub mod virtio_pci; pub use bus::{Bus, BusDevice, Device}; pub use gicv2::Gicv2; pub use gicv3::Gicv3; +pub use pci::PciHost; pub use pl011::{LEN as PL011_LEN, Pl011}; pub use virtio_blk::{BlockBackend, VirtioBlock, VirtioIo}; +pub use virtio_pci::VirtioBlockPci; diff --git a/src/emulator/src/devices/pci.rs b/src/emulator/src/devices/pci.rs new file mode 100644 index 0000000..5c912e2 --- /dev/null +++ b/src/emulator/src/devices/pci.rs @@ -0,0 +1,427 @@ +//! PCI for the `virt` board: the root bus's configuration space (ECAM), and the memory +//! BARs of the functions on it. +//! +//! Only bus 0 is populated, one function per slot, and no bridges sit below it. Firmware +//! assigns the BARs itself, as it does under QEMU, so a function claims the addresses its +//! BARs hold once the command register enables memory decoding. + +use alloc::boxed::Box; +use alloc::vec::Vec; + +/// The ECAM window of QEMU `virt`'s high PCIe region: 256 buses of 4 KiB functions. +pub const ECAM_BASE: u64 = 0x40_1000_0000; +pub const ECAM_SIZE: u64 = 0x1000_0000; + +/// The host bridge QEMU's GPEX reports at 00:00.0. +const HOST_BRIDGE_VENDOR: u16 = 0x1b36; +const HOST_BRIDGE_DEVICE: u16 = 0x0008; +/// Base class 0x06, bridge; subclass 0x00, host bridge. +const HOST_BRIDGE_CLASS: u32 = 0x06_00_00; + +const CONFIG_LEN: usize = 256; +pub const BAR_COUNT: usize = 6; +const BAR_OFFSET: usize = 0x10; +const CAPABILITIES_POINTER: usize = 0x34; +/// Capabilities start after the 64-byte standard header. +const FIRST_CAPABILITY: usize = 0x40; +const STATUS: usize = 0x06; +const STATUS_CAPABILITIES: u8 = 0x10; +const COMMAND: usize = 0x04; +/// The command bits a driver may set: I/O, memory, bus master, special cycles, MWI, +/// VGA palette snoop, parity, SERR, fast back-to-back and INTx disable. +const COMMAND_WRITABLE: [u8; 2] = [0x47, 0x05]; +const COMMAND_MEMORY: u8 = 0x2; + +/// The 256-byte configuration space of a type-0 function. Writes honour a per-byte +/// writable mask, and a memory BAR keeps only the address bits its size allows, so the +/// standard sizing probe reads back the size. +pub struct ConfigSpace { + bytes: [u8; CONFIG_LEN], + writable: [u8; CONFIG_LEN], + /// Size of each memory BAR in bytes, zero when the BAR is not implemented. + bar_size: [u64; BAR_COUNT], + /// Offset of the last capability, whose `next` pointer is filled in by the next one. + last_capability: Option, + /// The first offset free for another capability, four-byte aligned. + capabilities_end: usize, +} + +impl ConfigSpace { + /// `class` is base class, subclass and programming interface, as `0xBBSSPP`. + pub fn new( + vendor: u16, + device: u16, + class: u32, + subsystem_vendor: u16, + subsystem: u16, + ) -> Self { + let mut config = Self { + bytes: [0; CONFIG_LEN], + writable: [0; CONFIG_LEN], + bar_size: [0; BAR_COUNT], + last_capability: None, + capabilities_end: FIRST_CAPABILITY, + }; + config.bytes[0x00..0x02].copy_from_slice(&vendor.to_le_bytes()); + config.bytes[0x02..0x04].copy_from_slice(&device.to_le_bytes()); + config.bytes[0x09] = class as u8; + config.bytes[0x0a] = (class >> 8) as u8; + config.bytes[0x0b] = (class >> 16) as u8; + config.bytes[0x2c..0x2e].copy_from_slice(&subsystem_vendor.to_le_bytes()); + config.bytes[0x2e..0x30].copy_from_slice(&subsystem.to_le_bytes()); + config.writable[COMMAND] = COMMAND_WRITABLE[0]; + config.writable[COMMAND + 1] = COMMAND_WRITABLE[1]; + config + } + + /// Declares memory BAR `index` of `size` bytes, a power of two of at least 16. + pub fn set_bar(&mut self, index: usize, size: u64) { + assert!(index < BAR_COUNT && size.is_power_of_two() && (16..=1 << 32).contains(&size)); + self.bar_size[index] = size; + let at = BAR_OFFSET + 4 * index; + self.writable[at..at + 4].fill(0xff); + } + + /// Appends a capability with identifier `id`. `body` is what follows the `next` + /// pointer, starting with the capability's own length byte. + pub fn add_capability(&mut self, id: u8, body: &[u8]) { + let at = self.capabilities_end; + let length = 2 + body.len(); + assert!( + at + length <= CONFIG_LEN, + "capability does not fit configuration space" + ); + self.bytes[at] = id; + self.bytes[at + 1] = 0; + self.bytes[at + 2..at + length].copy_from_slice(body); + match self.last_capability { + None => self.bytes[CAPABILITIES_POINTER] = at as u8, + Some(previous) => self.bytes[previous + 1] = at as u8, + } + self.bytes[STATUS] |= STATUS_CAPABILITIES; + self.last_capability = Some(at); + self.capabilities_end = (at + length + 3) & !3; + } + + /// The base and size of memory BAR `index`, once firmware has assigned it. + pub fn memory_bar(&self, index: usize) -> Option<(u64, u64)> { + let size = self.bar_size[index]; + if size == 0 { + return None; + } + let at = BAR_OFFSET + 4 * index; + let base = u32::from_le_bytes(self.bytes[at..at + 4].try_into().unwrap()); + Some((u64::from(base & !0xf), size)) + } + + /// Whether the command register lets the function decode its memory BARs. + pub fn memory_decoding(&self) -> bool { + self.bytes[COMMAND] & COMMAND_MEMORY != 0 + } + + /// Reads `size` bytes, little-endian. Bytes past the header read as zero. + pub fn read(&self, offset: usize, size: usize) -> u64 { + (0..size).fold(0, |value, i| { + let byte = self.bytes.get(offset + i).copied().unwrap_or(0); + value | u64::from(byte) << (8 * i) + }) + } + + /// Writes `size` bytes, little-endian, keeping the bytes the writable mask protects. + pub fn write(&mut self, offset: usize, size: usize, value: u64) { + for i in 0..size { + let at = offset + i; + if at >= CONFIG_LEN { + break; + } + let mask = self.writable[at]; + let byte = (value >> (8 * i)) as u8; + self.bytes[at] = (self.bytes[at] & !mask) | (byte & mask); + } + for index in 0..BAR_COUNT { + let size = self.bar_size[index]; + if size == 0 { + continue; + } + let at = BAR_OFFSET + 4 * index; + let written = u32::from_le_bytes(self.bytes[at..at + 4].try_into().unwrap()); + let kept = written & !((size - 1) as u32) & !0xf; + self.bytes[at..at + 4].copy_from_slice(&kept.to_le_bytes()); + } + } +} + +/// A function on the root bus. Its BARs are reached through the bus, which decides which +/// function a bus address belongs to. +pub trait PciFunction { + fn config(&self) -> &ConfigSpace; + fn config_mut(&mut self) -> &mut ConfigSpace; + /// Reads `size` bytes at `offset` into memory BAR `bar`. + fn bar_read(&mut self, bar: usize, offset: u64, size: u8) -> u64 { + let _ = (bar, offset, size); + 0 + } + /// Writes `size` bytes at `offset` into memory BAR `bar`. + fn bar_write(&mut self, bar: usize, offset: u64, size: u8, value: u64) { + let _ = (bar, offset, size, value); + } +} + +/// The root bus: functions attached to slots, served through ECAM and their BARs. +#[derive(Default)] +pub struct PciHost { + slots: Vec<(u8, Box)>, +} + +impl PciHost { + /// A bus with the GPEX host bridge at 00:00.0 and nothing else. + pub fn new() -> Self { + let mut host = Self::default(); + host.attach(0, Box::new(HostBridge::new())); + host + } + + /// Attaches `function` at `slot` on bus 0. + pub fn attach(&mut self, slot: u8, function: Box) { + assert!(slot < 32, "slot {slot} is not on bus 0"); + assert!( + self.slots.iter().all(|(taken, _)| *taken != slot), + "slot {slot} is already occupied" + ); + self.slots.push((slot, function)); + } + + /// Reads `size` bytes at a physical address the bus claims, or `None` for one it does not. + pub fn read(&mut self, address: u64, size: u8) -> Option { + if let Some(ecam) = ecam_offset(address) { + return Some(self.config_read(ecam, size)); + } + let (index, bar, offset) = self.claim(address)?; + Some(self.slots[index].1.bar_read(bar, offset, size)) + } + + /// Writes `size` bytes at a physical address the bus claims. `false` declines one it does not. + pub fn write(&mut self, address: u64, size: u8, value: u64) -> bool { + if let Some(ecam) = ecam_offset(address) { + self.config_write(ecam, size, value); + return true; + } + let Some((index, bar, offset)) = self.claim(address) else { + return false; + }; + self.slots[index].1.bar_write(bar, offset, size, value); + true + } + + fn config_read(&self, ecam: u64, size: u8) -> u64 { + let all_ones = size_mask(size); + match self.function(ecam) { + Some((function, register)) => function.config().read(register, size as usize), + None => all_ones, + } + } + + fn config_write(&mut self, ecam: u64, size: u8, value: u64) { + let Some(slot) = self.function_index(ecam) else { + return; + }; + let register = (ecam & 0xfff) as usize; + self.slots[slot] + .1 + .config_mut() + .write(register, size as usize, value); + } + + /// The function an ECAM offset names, and the register within it. Only bus 0 and + /// function 0 are populated. + fn function(&self, ecam: u64) -> Option<(&dyn PciFunction, usize)> { + let index = self.function_index(ecam)?; + Some((self.slots[index].1.as_ref(), (ecam & 0xfff) as usize)) + } + + fn function_index(&self, ecam: u64) -> Option { + let bus = ecam >> 20; + let slot = ((ecam >> 15) & 0x1f) as u8; + let function = (ecam >> 12) & 0x7; + if bus != 0 || function != 0 { + return None; + } + self.slots.iter().position(|(taken, _)| *taken == slot) + } + + /// The function, BAR and offset a bus address falls in. Only functions that enable + /// memory decoding answer. + fn claim(&self, address: u64) -> Option<(usize, usize, u64)> { + for (index, (_, function)) in self.slots.iter().enumerate() { + let config = function.config(); + if !config.memory_decoding() { + continue; + } + for bar in 0..BAR_COUNT { + if let Some((base, size)) = config.memory_bar(bar) + && (base..base + size).contains(&address) + { + return Some((index, bar, address - base)); + } + } + } + None + } +} + +fn ecam_offset(address: u64) -> Option { + (ECAM_BASE..ECAM_BASE + ECAM_SIZE) + .contains(&address) + .then(|| address - ECAM_BASE) +} + +fn size_mask(size: u8) -> u64 { + if size >= 8 { + u64::MAX + } else { + (1u64 << (8 * size)) - 1 + } +} + +/// The GPEX host bridge at 00:00.0. It has no BARs; its configuration space is what +/// enumeration reads first. +pub struct HostBridge { + config: ConfigSpace, +} + +impl HostBridge { + fn new() -> Self { + Self { + config: ConfigSpace::new( + HOST_BRIDGE_VENDOR, + HOST_BRIDGE_DEVICE, + HOST_BRIDGE_CLASS, + 0, + 0, + ), + } + } +} + +impl PciFunction for HostBridge { + fn config(&self) -> &ConfigSpace { + &self.config + } + + fn config_mut(&mut self) -> &mut ConfigSpace { + &mut self.config + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::rc::Rc; + use core::cell::RefCell; + + type Writes = Rc>>; + + /// A function with one 16 KiB BAR. Its BAR reads return the offset, and its writes are + /// recorded where the test can see them. + struct Probe { + config: ConfigSpace, + writes: Writes, + } + + impl Probe { + fn new(writes: Writes) -> Self { + let mut config = ConfigSpace::new(0x1af4, 0x1042, 0x018000, 0x1af4, 2); + config.set_bar(0, 0x4000); + // A capability, so the status register's capability-list bit is set as on virtio. + let mut body = [0u8; 14]; + body[0] = 16; + config.add_capability(0x09, &body); + Self { config, writes } + } + } + + impl PciFunction for Probe { + fn config(&self) -> &ConfigSpace { + &self.config + } + fn config_mut(&mut self) -> &mut ConfigSpace { + &mut self.config + } + fn bar_read(&mut self, bar: usize, offset: u64, _size: u8) -> u64 { + if bar == 0 { offset } else { 0 } + } + fn bar_write(&mut self, bar: usize, offset: u64, size: u8, value: u64) { + self.writes.borrow_mut().push((bar, offset, size, value)); + } + } + + fn ecam(bus: u64, slot: u64, function: u64, register: u64) -> u64 { + ECAM_BASE | bus << 20 | slot << 15 | function << 12 | register + } + + fn probe_host() -> (PciHost, Writes) { + let writes = Writes::default(); + let mut host = PciHost::new(); + host.attach(1, Box::new(Probe::new(writes.clone()))); + (host, writes) + } + + #[test] + fn host_bridge_identifies_itself_and_empty_slots_read_as_all_ones() { + let mut host = PciHost::new(); + assert_eq!( + host.read(ecam(0, 0, 0, 0), 4), + Some(u64::from(HOST_BRIDGE_VENDOR) | u64::from(HOST_BRIDGE_DEVICE) << 16) + ); + assert_eq!(host.read(ecam(0, 5, 0, 0), 4), Some(0xffff_ffff)); + assert_eq!(host.read(ecam(0, 5, 0, 0), 2), Some(0xffff)); + assert_eq!(host.read(ecam(1, 0, 0, 0), 4), Some(0xffff_ffff)); + assert_eq!(host.read(ecam(0, 0, 3, 0), 4), Some(0xffff_ffff)); + } + + #[test] + fn bar_sizing_probe_reads_back_the_size_mask_and_restores_the_address() { + let (mut host, _) = probe_host(); + let bar = ecam(0, 1, 0, 0x10); + host.write(bar, 4, 0x1000_0000); + assert_eq!(host.read(bar, 4), Some(0x1000_0000)); + // All ones reads back as the writable address bits, which name 16 KiB. + host.write(bar, 4, 0xffff_ffff); + assert_eq!(host.read(bar, 4), Some(0xffff_c000)); + host.write(bar, 4, 0x1000_0000); + assert_eq!(host.read(bar, 4), Some(0x1000_0000)); + } + + #[test] + fn a_bar_answers_only_while_memory_decoding_is_enabled() { + let (mut host, _) = probe_host(); + host.write(ecam(0, 1, 0, 0x10), 4, 0x1000_0000); + assert_eq!(host.read(0x1000_0004, 4), None); + host.write(ecam(0, 1, 0, 0x04), 2, 0x0002); + assert_eq!(host.read(0x1000_0004, 4), Some(4)); + assert_eq!( + host.read(0x1000_4000, 4), + None, + "one past the BAR is unclaimed" + ); + assert!(!host.write(0x1000_4000, 4, 0)); + } + + #[test] + fn a_write_into_a_bar_reaches_the_function_at_its_offset() { + let (mut host, writes) = probe_host(); + host.write(ecam(0, 1, 0, 0x10), 4, 0x1000_0000); + host.write(ecam(0, 1, 0, 0x04), 2, 0x0002); + assert!(host.write(0x1000_0040, 2, 0xabcd)); + assert_eq!(writes.borrow().as_slice(), &[(0, 0x40, 2, 0xabcd)]); + } + + #[test] + fn the_command_register_keeps_only_writable_bits_and_status_lists_capabilities() { + let (mut host, _) = probe_host(); + // Status is read only: its capability-list bit (bit 4) survives the write. + host.write(ecam(0, 1, 0, 0x04), 4, 0xffff_ffff); + assert_eq!(host.read(ecam(0, 1, 0, 0x04), 2), Some(0x0547)); + assert_eq!(host.read(ecam(0, 1, 0, 0x06), 2), Some(0x0010)); + } +} diff --git a/src/emulator/src/devices/virtio_blk.rs b/src/emulator/src/devices/virtio_blk.rs index bed07b9..ea827ca 100644 --- a/src/emulator/src/devices/virtio_blk.rs +++ b/src/emulator/src/devices/virtio_blk.rs @@ -1,5 +1,9 @@ -//! VirtIO-MMIO v2 Block Device (VIRTIO_ID_BLOCK). -//! Implements the VirtIO 1.0 (v2 modern) memory-mapped block device specification. +//! VirtIO block device: the request queue, shared by its transports. +//! +//! `VirtioBlock` is the VirtIO-MMIO v2 transport (the Linux virt board's `virtio,mmio` +//! device). `virtio_pci::VirtioBlockPci` is the same device behind the modern PCI transport. +//! Both own the register state and hand the queue to a [`BlockQueue`], which serves the +//! requests the driver made available. use crate::memory::{GuestMemory, SharedMemory}; use alloc::vec::Vec; @@ -9,10 +13,10 @@ pub const VERSION: u32 = 2; // Modern VirtIO v2 pub const DEVICE_ID_BLOCK: u32 = 2; // Block device pub const VENDOR_ID: u32 = 0x554d4551; // "QEMU" -// Feature bits +// Feature bits (virtio 1.2, section 5.2.3) pub const VIRTIO_BLK_F_RO: u64 = 1 << 5; pub const VIRTIO_BLK_F_BLK_SIZE: u64 = 1 << 6; -pub const VIRTIO_BLK_F_FLUSH: u64 = 1 << 11; +pub const VIRTIO_BLK_F_FLUSH: u64 = 1 << 9; pub const VIRTIO_F_VERSION_1: u64 = 1 << 32; // Request types @@ -30,6 +34,8 @@ pub const VRING_DESC_F_NEXT: u16 = 1; pub const VRING_DESC_F_WRITE: u16 = 2; pub const QUEUE_MAX_SIZE: u32 = 128; +/// Block addresses are in sectors of this many bytes. +pub const SECTOR_SIZE: u64 = 512; /// Storage backend for a block device. pub trait BlockBackend: Send { @@ -72,26 +78,206 @@ impl BlockBackend for Vec { Ok(()) } } + +/// The virtqueue a transport configures: its size, its three rings, and the progress +/// the device has made through them. #[derive(Clone, Copy, Default)] -struct Queue { - num: u32, - ready: bool, - desc_table: u64, - avail_ring: u64, - used_ring: u64, - last_avail_idx: u16, - last_used_idx: u16, +pub struct Queue { + pub num: u32, + pub ready: bool, + pub desc_table: u64, + pub avail_ring: u64, + pub used_ring: u64, + pub last_avail_idx: u16, + pub last_used_idx: u16, } -pub struct VirtioBlock { +/// The backing store, the guest memory the rings and buffers live in, and the one request +/// queue. Transports set `queue` from their registers and call [`BlockQueue::process`] +/// when the driver kicks it. +pub struct BlockQueue { pub backend: B, memory: SharedMemory, + pub queue: Queue, +} + +impl BlockQueue { + pub fn new(backend: B, memory: SharedMemory) -> Self { + Self { + backend, + memory, + queue: Queue::default(), + } + } + + /// The disk's capacity in sectors, as the device configuration reports it. + pub fn capacity_sectors(&self) -> u64 { + self.backend.len() / SECTOR_SIZE + } + + /// Serves every request the driver made available. Returns whether any was served. + pub fn process(&mut self) -> bool { + if self.queue.num == 0 || !self.queue.ready { + return false; + } + + let avail_addr = self.queue.avail_ring; + let desc_addr = self.queue.desc_table; + let used_addr = self.queue.used_ring; + let q_num = self.queue.num as u64; + + // Driver increments idx in avail ring + let avail_idx = self.read_guest_u16(avail_addr + 2); + let mut processed_any = false; + + while self.queue.last_avail_idx != avail_idx { + let ring_slot = (self.queue.last_avail_idx as u64) % q_num; + let head_idx = self.read_guest_u16(avail_addr + 4 + ring_slot * 2) as u64; + + // Walk descriptor chain + let mut curr = head_idx; + let mut req_header_addr = None; + let mut data_buffers: Vec<(u64, u32, bool)> = Vec::new(); + let mut status_addr = None; + + for _ in 0..q_num { + let d_addr = desc_addr + curr * 16; + let addr = self.read_guest_u64(d_addr); + let len = self.read_guest_u32(d_addr + 8); + let flags = self.read_guest_u16(d_addr + 12); + let next = self.read_guest_u16(d_addr + 14) as u64; + + let is_write = (flags & VRING_DESC_F_WRITE) != 0; + let has_next = (flags & VRING_DESC_F_NEXT) != 0; + + if req_header_addr.is_none() { + // First descriptor: virtio_blk_req header + req_header_addr = Some(addr); + } else if !has_next && is_write && len == 1 { + // Last descriptor in chain: status byte + status_addr = Some(addr); + } else { + // Data buffer + data_buffers.push((addr, len, is_write)); + } + + if !has_next { + break; + } + curr = next; + } + + let mut total_written = 0u32; + let mut status = VIRTIO_BLK_S_OK; + + if let Some(req_addr) = req_header_addr { + let req_type = self.read_guest_u32(req_addr); + let sector = self.read_guest_u64(req_addr + 8); + let mut disk_offset = sector.saturating_mul(SECTOR_SIZE); + + match req_type { + VIRTIO_BLK_T_IN => { + // Read from disk to guest memory + for (buf_addr, len, _) in data_buffers { + let mut tmp = alloc::vec![0u8; len as usize]; + if self.backend.read_at(disk_offset, &mut tmp).is_ok() { + let _ = self.memory.write(buf_addr, &tmp); + total_written = total_written.saturating_add(len); + } else { + status = VIRTIO_BLK_S_IOERR; + } + disk_offset = disk_offset.saturating_add(len as u64); + } + } + VIRTIO_BLK_T_OUT => { + // Write from guest memory to disk + for (buf_addr, len, _) in data_buffers { + let mut tmp = alloc::vec![0u8; len as usize]; + let _ = self.memory.read(buf_addr, &mut tmp); + if self.backend.write_at(disk_offset, &tmp).is_err() { + status = VIRTIO_BLK_S_IOERR; + } + disk_offset = disk_offset.saturating_add(len as u64); + } + } + VIRTIO_BLK_T_FLUSH => { + if self.backend.flush().is_err() { + status = VIRTIO_BLK_S_IOERR; + } + } + VIRTIO_BLK_T_GET_ID => { + for (buf_addr, len, _) in data_buffers { + let id = b"volt-blk\0"; + let copy_len = (len as usize).min(id.len()); + let _ = self.memory.write(buf_addr, &id[..copy_len]); + total_written = total_written.saturating_add(copy_len as u32); + } + } + _ => { + status = VIRTIO_BLK_S_IOERR; + } + } + } else { + status = VIRTIO_BLK_S_IOERR; + } + + // Write status byte (1 byte, written by device) + if let Some(s_addr) = status_addr { + let _ = self.memory.write(s_addr, &[status]); + total_written = total_written.saturating_add(1); + } + + // Write entry to used ring + let used_slot = (self.queue.last_used_idx as u64) % q_num; + let elem_addr = used_addr + 4 + used_slot * 8; + self.write_guest_u32(elem_addr, head_idx as u32); + self.write_guest_u32(elem_addr + 4, total_written); + + self.queue.last_used_idx = self.queue.last_used_idx.wrapping_add(1); + self.write_guest_u16(used_addr + 2, self.queue.last_used_idx); + + self.queue.last_avail_idx = self.queue.last_avail_idx.wrapping_add(1); + processed_any = true; + } + + processed_any + } + + fn read_guest_u16(&self, addr: u64) -> u16 { + let mut buf = [0u8; 2]; + let _ = self.memory.read(addr, &mut buf); + u16::from_le_bytes(buf) + } + + fn read_guest_u32(&self, addr: u64) -> u32 { + let mut buf = [0u8; 4]; + let _ = self.memory.read(addr, &mut buf); + u32::from_le_bytes(buf) + } + + fn read_guest_u64(&self, addr: u64) -> u64 { + let mut buf = [0u8; 8]; + let _ = self.memory.read(addr, &mut buf); + u64::from_le_bytes(buf) + } + + fn write_guest_u16(&mut self, addr: u64, val: u16) { + let _ = self.memory.write(addr, &val.to_le_bytes()); + } + + fn write_guest_u32(&mut self, addr: u64, val: u32) { + let _ = self.memory.write(addr, &val.to_le_bytes()); + } +} + +/// The VirtIO-MMIO v2 transport for the block device. +pub struct VirtioBlock { + core: BlockQueue, status: u32, device_features_sel: u32, driver_features_sel: u32, driver_features: u64, queue_sel: u32, - queue: Queue, interrupt_status: u32, line: Option, } @@ -99,14 +285,12 @@ pub struct VirtioBlock { impl VirtioBlock { pub fn new(backend: B, memory: SharedMemory) -> Self { Self { - backend, - memory, + core: BlockQueue::new(backend, memory), status: 0, device_features_sel: 0, driver_features_sel: 0, driver_features: 0, queue_sel: 0, - queue: Queue::default(), interrupt_status: 0, line: None, } @@ -116,14 +300,12 @@ impl VirtioBlock { impl VirtioBlock { pub fn with_line(self, line: L2) -> VirtioBlock { VirtioBlock { - backend: self.backend, - memory: self.memory, + core: self.core, status: self.status, device_features_sel: self.device_features_sel, driver_features_sel: self.driver_features_sel, driver_features: self.driver_features, queue_sel: self.queue_sel, - queue: self.queue, interrupt_status: self.interrupt_status, line: Some(line), } @@ -165,7 +347,7 @@ impl VirtioBlock { } } 0x044 => { - if self.queue_sel == 0 && self.queue.ready { + if self.queue_sel == 0 && self.core.queue.ready { 1 } else { 0 @@ -177,7 +359,7 @@ impl VirtioBlock { // Configuration space: // 0x100..0x108: capacity in 512-byte sectors (u64) 0x100..=0x107 => { - let capacity = self.backend.len() / 512; + let capacity = self.core.capacity_sectors(); let shift = (offset - 0x100) * 8; capacity >> shift } @@ -210,18 +392,18 @@ impl VirtioBlock { 0x030 => self.queue_sel = value as u32, 0x038 => { if self.queue_sel == 0 { - self.queue.num = (value as u32).min(QUEUE_MAX_SIZE); + self.core.queue.num = (value as u32).min(QUEUE_MAX_SIZE); } } 0x044 => { if self.queue_sel == 0 { - self.queue.ready = (value & 1) != 0; + self.core.queue.ready = (value & 1) != 0; } } 0x050 => { let queue_idx = value as u32; - if queue_idx == 0 && self.queue.ready { - self.process_queue(); + if queue_idx == 0 && self.core.queue.ready && self.core.process() { + self.raise_irq(); } } 0x064 => { @@ -234,183 +416,39 @@ impl VirtioBlock { self.status = value as u32; if self.status == 0 { // Reset - self.queue = Queue::default(); + self.core.queue = Queue::default(); self.lower_irq(); } } 0x080 => { - self.queue.desc_table = - (self.queue.desc_table & !0xffff_ffff) | (value & 0xffff_ffff) + self.core.queue.desc_table = + (self.core.queue.desc_table & !0xffff_ffff) | (value & 0xffff_ffff) + } + 0x084 => { + self.core.queue.desc_table = + (self.core.queue.desc_table & 0xffff_ffff) | (value << 32) } - 0x084 => self.queue.desc_table = (self.queue.desc_table & 0xffff_ffff) | (value << 32), 0x090 => { - self.queue.avail_ring = - (self.queue.avail_ring & !0xffff_ffff) | (value & 0xffff_ffff) + self.core.queue.avail_ring = + (self.core.queue.avail_ring & !0xffff_ffff) | (value & 0xffff_ffff) } - 0x094 => self.queue.avail_ring = (self.queue.avail_ring & 0xffff_ffff) | (value << 32), - 0x0a0 => { - self.queue.used_ring = (self.queue.used_ring & !0xffff_ffff) | (value & 0xffff_ffff) + 0x094 => { + self.core.queue.avail_ring = + (self.core.queue.avail_ring & 0xffff_ffff) | (value << 32) } - 0x0a4 => self.queue.used_ring = (self.queue.used_ring & 0xffff_ffff) | (value << 32), - _ => {} - } - } - - fn read_guest_u16(&self, addr: u64) -> u16 { - let mut buf = [0u8; 2]; - let _ = self.memory.read(addr, &mut buf); - u16::from_le_bytes(buf) - } - - fn read_guest_u32(&self, addr: u64) -> u32 { - let mut buf = [0u8; 4]; - let _ = self.memory.read(addr, &mut buf); - u32::from_le_bytes(buf) - } - - fn read_guest_u64(&self, addr: u64) -> u64 { - let mut buf = [0u8; 8]; - let _ = self.memory.read(addr, &mut buf); - u64::from_le_bytes(buf) - } - - fn write_guest_u16(&mut self, addr: u64, val: u16) { - let _ = self.memory.write(addr, &val.to_le_bytes()); - } - - fn write_guest_u32(&mut self, addr: u64, val: u32) { - let _ = self.memory.write(addr, &val.to_le_bytes()); - } - - fn process_queue(&mut self) { - if self.queue.num == 0 || !self.queue.ready { - return; - } - - let avail_addr = self.queue.avail_ring; - let desc_addr = self.queue.desc_table; - let used_addr = self.queue.used_ring; - let q_num = self.queue.num as u64; - - // Driver increments idx in avail ring - let avail_idx = self.read_guest_u16(avail_addr + 2); - let mut processed_any = false; - - while self.queue.last_avail_idx != avail_idx { - let ring_slot = (self.queue.last_avail_idx as u64) % q_num; - let head_idx = self.read_guest_u16(avail_addr + 4 + ring_slot * 2) as u64; - - // Walk descriptor chain - let mut curr = head_idx; - let mut req_header_addr = None; - let mut data_buffers: Vec<(u64, u32, bool)> = Vec::new(); - let mut status_addr = None; - - for _ in 0..q_num { - let d_addr = desc_addr + curr * 16; - let addr = self.read_guest_u64(d_addr); - let len = self.read_guest_u32(d_addr + 8); - let flags = self.read_guest_u16(d_addr + 12); - let next = self.read_guest_u16(d_addr + 14) as u64; - - let is_write = (flags & VRING_DESC_F_WRITE) != 0; - let has_next = (flags & VRING_DESC_F_NEXT) != 0; - - if req_header_addr.is_none() { - // First descriptor: virtio_blk_req header - req_header_addr = Some(addr); - } else if !has_next && is_write && len == 1 { - // Last descriptor in chain: status byte - status_addr = Some(addr); - } else { - // Data buffer - data_buffers.push((addr, len, is_write)); - } - - if !has_next { - break; - } - curr = next; - } - - let mut total_written = 0u32; - let mut status = VIRTIO_BLK_S_OK; - - if let Some(req_addr) = req_header_addr { - let req_type = self.read_guest_u32(req_addr); - let sector = self.read_guest_u64(req_addr + 8); - let mut disk_offset = sector.saturating_mul(512); - - match req_type { - VIRTIO_BLK_T_IN => { - // Read from disk to guest memory - for (buf_addr, len, _) in data_buffers { - let mut tmp = alloc::vec![0u8; len as usize]; - if self.backend.read_at(disk_offset, &mut tmp).is_ok() { - let _ = self.memory.write(buf_addr, &tmp); - total_written = total_written.saturating_add(len); - } else { - status = VIRTIO_BLK_S_IOERR; - } - disk_offset = disk_offset.saturating_add(len as u64); - } - } - VIRTIO_BLK_T_OUT => { - // Write from guest memory to disk - for (buf_addr, len, _) in data_buffers { - let mut tmp = alloc::vec![0u8; len as usize]; - let _ = self.memory.read(buf_addr, &mut tmp); - if self.backend.write_at(disk_offset, &tmp).is_err() { - status = VIRTIO_BLK_S_IOERR; - } - disk_offset = disk_offset.saturating_add(len as u64); - } - } - VIRTIO_BLK_T_FLUSH => { - if self.backend.flush().is_err() { - status = VIRTIO_BLK_S_IOERR; - } - } - VIRTIO_BLK_T_GET_ID => { - for (buf_addr, len, _) in data_buffers { - let id = b"volt-blk\0"; - let copy_len = (len as usize).min(id.len()); - let _ = self.memory.write(buf_addr, &id[..copy_len]); - total_written = total_written.saturating_add(copy_len as u32); - } - } - _ => { - status = VIRTIO_BLK_S_IOERR; - } - } - } else { - status = VIRTIO_BLK_S_IOERR; + 0x0a0 => { + self.core.queue.used_ring = + (self.core.queue.used_ring & !0xffff_ffff) | (value & 0xffff_ffff) } - - // Write status byte (1 byte, written by device) - if let Some(s_addr) = status_addr { - let _ = self.memory.write(s_addr, &[status]); - total_written = total_written.saturating_add(1); + 0x0a4 => { + self.core.queue.used_ring = + (self.core.queue.used_ring & 0xffff_ffff) | (value << 32) } - - // Write entry to used ring - let used_slot = (self.queue.last_used_idx as u64) % q_num; - let elem_addr = used_addr + 4 + used_slot * 8; - self.write_guest_u32(elem_addr, head_idx as u32); - self.write_guest_u32(elem_addr + 4, total_written); - - self.queue.last_used_idx = self.queue.last_used_idx.wrapping_add(1); - self.write_guest_u16(used_addr + 2, self.queue.last_used_idx); - - self.queue.last_avail_idx = self.queue.last_avail_idx.wrapping_add(1); - processed_any = true; - } - - if processed_any { - self.raise_irq(); + _ => {} } } } + pub trait VirtioIo { fn read(&mut self, offset: u64, size: u8) -> u64; fn write(&mut self, offset: u64, size: u8, value: u64); diff --git a/src/emulator/src/devices/virtio_pci.rs b/src/emulator/src/devices/virtio_pci.rs new file mode 100644 index 0000000..9747397 --- /dev/null +++ b/src/emulator/src/devices/virtio_pci.rs @@ -0,0 +1,535 @@ +//! VirtIO block device on PCI: the modern (virtio 1.0) transport. +//! +//! The device is a type-0 function, vendor 0x1af4, device 0x1042 (0x1040 plus the block +//! type). BAR0 holds the four virtio structures, each at its own offset, and the capability +//! list names them: +//! +//! | structure | offset | length | notes | +//! |-----------|--------|--------|------------------------------------| +//! | common | 0x0000 | 0x40 | feature, status, queue registers | +//! | notify | 0x1000 | 0x100 | a write of a queue index kicks it | +//! | ISR | 0x2000 | 0x4 | never raised: the firmware polls | +//! | device | 0x3000 | 0x100 | the block configuration | +//! +//! Requests are served by the same `BlockQueue` the MMIO transport uses. + +use super::pci::{ConfigSpace, PciFunction}; +use super::virtio_blk::{ + BlockBackend, BlockQueue, QUEUE_MAX_SIZE, Queue, SECTOR_SIZE, VIRTIO_BLK_F_BLK_SIZE, + VIRTIO_BLK_F_FLUSH, VIRTIO_F_VERSION_1, +}; +use crate::memory::SharedMemory; +use alloc::vec::Vec; + +const VIRTIO_VENDOR: u16 = 0x1af4; +/// The modern device ID: 0x1040 plus the virtio device type, 2 for block. +const DEVICE_ID_BLOCK: u16 = 0x1042; +/// The subsystem device ID carries the device type for transitional drivers. +const SUBSYSTEM_BLOCK: u16 = 2; +/// Base class 0x01, mass storage; subclass 0x80, other. +const CLASS_STORAGE_OTHER: u32 = 0x01_80_00; + +/// The BAR holding every structure, and its size. +const BAR: usize = 0; +const BAR_SIZE: u64 = 0x4000; + +const COMMON_OFFSET: u64 = 0x0000; +const COMMON_LEN: u64 = 0x40; +const NOTIFY_OFFSET: u64 = 0x1000; +const NOTIFY_LEN: u64 = 0x100; +/// Each queue's notify register sits at `queue_notify_off * NOTIFY_MULTIPLIER` into the region. +const NOTIFY_MULTIPLIER: u32 = 4; +const ISR_OFFSET: u64 = 0x2000; +const ISR_LEN: u64 = 0x4; +const DEVICE_OFFSET: u64 = 0x3000; +const DEVICE_LEN: u64 = 0x100; + +/// Capability identifier of a vendor-specific capability. +const CAP_VENDOR: u8 = 0x09; +const CFG_COMMON: u8 = 1; +const CFG_NOTIFY: u8 = 2; +const CFG_ISR: u8 = 3; +const CFG_DEVICE: u8 = 4; + +/// `MSI-X` vector meaning "none". +const NO_VECTOR: u16 = 0xffff; +const NUM_QUEUES: u16 = 1; +/// The features offered. Only the features this model implements are offered. +const FEATURES: u64 = VIRTIO_F_VERSION_1 | VIRTIO_BLK_F_BLK_SIZE | VIRTIO_BLK_F_FLUSH; + +/// Common configuration offsets, from virtio 1.2 section 4.1.4.3. +mod common { + pub const DEVICE_FEATURE_SELECT: u64 = 0x00; + pub const DEVICE_FEATURE: u64 = 0x04; + pub const DRIVER_FEATURE_SELECT: u64 = 0x08; + pub const DRIVER_FEATURE: u64 = 0x0c; + pub const MSIX_CONFIG: u64 = 0x10; + pub const DEVICE_STATUS: u64 = 0x14; + pub const QUEUE_SELECT: u64 = 0x16; + pub const QUEUE_SIZE: u64 = 0x18; + pub const QUEUE_MSIX_VECTOR: u64 = 0x1a; + pub const QUEUE_ENABLE: u64 = 0x1c; + pub const QUEUE_DESC: u64 = 0x20; + pub const QUEUE_DRIVER: u64 = 0x28; + pub const QUEUE_DEVICE: u64 = 0x30; +} + +/// A virtio block device on the modern PCI transport. +pub struct VirtioBlockPci { + config: ConfigSpace, + core: BlockQueue, + device_features_sel: u32, + driver_features_sel: u32, + driver_features: u64, + status: u8, + queue_select: u16, + msix_config: u16, + queue_msix_vector: u16, +} + +impl VirtioBlockPci { + pub fn new(backend: B, memory: SharedMemory) -> Self { + let mut config = ConfigSpace::new( + VIRTIO_VENDOR, + DEVICE_ID_BLOCK, + CLASS_STORAGE_OTHER, + VIRTIO_VENDOR, + SUBSYSTEM_BLOCK, + ); + config.set_bar(BAR, BAR_SIZE); + config.add_capability( + CAP_VENDOR, + &structure_body(CFG_COMMON, COMMON_OFFSET, COMMON_LEN, None), + ); + config.add_capability( + CAP_VENDOR, + &structure_body( + CFG_NOTIFY, + NOTIFY_OFFSET, + NOTIFY_LEN, + Some(NOTIFY_MULTIPLIER), + ), + ); + config.add_capability( + CAP_VENDOR, + &structure_body(CFG_ISR, ISR_OFFSET, ISR_LEN, None), + ); + config.add_capability( + CAP_VENDOR, + &structure_body(CFG_DEVICE, DEVICE_OFFSET, DEVICE_LEN, None), + ); + Self { + config, + core: BlockQueue::new(backend, memory), + device_features_sel: 0, + driver_features_sel: 0, + driver_features: 0, + status: 0, + queue_select: 0, + msix_config: NO_VECTOR, + queue_msix_vector: NO_VECTOR, + } + } + + fn reset(&mut self) { + self.device_features_sel = 0; + self.driver_features_sel = 0; + self.driver_features = 0; + self.queue_select = 0; + self.msix_config = NO_VECTOR; + self.queue_msix_vector = NO_VECTOR; + self.core.queue = Queue::default(); + } + + /// A 32-bit word of the common configuration, at a four-byte-aligned offset. + fn common_dword(&self, at: u64) -> u32 { + let queue = &self.core.queue; + let selected = self.queue_select == 0; + match at { + common::DEVICE_FEATURE_SELECT => self.device_features_sel, + common::DEVICE_FEATURE => match self.device_features_sel { + 0 => FEATURES as u32, + 1 => (FEATURES >> 32) as u32, + _ => 0, + }, + common::DRIVER_FEATURE_SELECT => self.driver_features_sel, + common::DRIVER_FEATURE => match self.driver_features_sel { + 0 => self.driver_features as u32, + 1 => (self.driver_features >> 32) as u32, + _ => 0, + }, + // msix_config, then num_queues. + 0x10 => u32::from(self.msix_config) | u32::from(NUM_QUEUES) << 16, + // device_status, config_generation (always 0), then queue_select. + 0x14 => u32::from(self.status) | u32::from(self.queue_select) << 16, + // queue_size (the maximum until the driver sets one), then queue_msix_vector. + 0x18 => { + let size = match (selected, queue.num) { + (false, _) => 0, + (true, 0) => QUEUE_MAX_SIZE, + (true, num) => num, + }; + size | u32::from(self.queue_msix_vector) << 16 + } + // queue_enable, then queue_notify_off, which is the queue index. + 0x1c => { + let enable = u32::from(selected && queue.ready); + enable | u32::from(self.queue_select) << 16 + } + common::QUEUE_DESC => queue.desc_table as u32, + 0x24 => (queue.desc_table >> 32) as u32, + common::QUEUE_DRIVER => queue.avail_ring as u32, + 0x2c => (queue.avail_ring >> 32) as u32, + common::QUEUE_DEVICE => queue.used_ring as u32, + 0x34 => (queue.used_ring >> 32) as u32, + _ => 0, + } + } + + /// Writes one field of the common configuration. Fields are written at their own + /// offsets; a 64-bit field arrives as one access or as two halves. + fn common_write(&mut self, at: u64, size: u8, value: u64) { + if size == 8 { + self.common_write(at, 4, value & 0xffff_ffff); + self.common_write(at + 4, 4, value >> 32); + return; + } + let low = value as u32; + match at { + common::DEVICE_FEATURE_SELECT => self.device_features_sel = low, + common::DRIVER_FEATURE_SELECT => self.driver_features_sel = low, + common::DRIVER_FEATURE => match self.driver_features_sel { + 0 => { + self.driver_features = (self.driver_features & !0xffff_ffff) | u64::from(low); + } + 1 => { + self.driver_features = + (self.driver_features & 0xffff_ffff) | u64::from(low) << 32; + } + _ => {} + }, + common::MSIX_CONFIG => self.msix_config = value as u16, + common::DEVICE_STATUS => { + self.status = value as u8; + if self.status == 0 { + self.reset(); + } + } + common::QUEUE_SELECT => self.queue_select = value as u16, + common::QUEUE_SIZE => { + if self.queue_select == 0 { + self.core.queue.num = (value as u32).min(QUEUE_MAX_SIZE); + } + } + common::QUEUE_MSIX_VECTOR => self.queue_msix_vector = value as u16, + common::QUEUE_ENABLE => { + if self.queue_select == 0 { + self.core.queue.ready = value & 1 != 0; + } + } + common::QUEUE_DESC => set_low(&mut self.core.queue.desc_table, low), + 0x24 => set_high(&mut self.core.queue.desc_table, low), + common::QUEUE_DRIVER => set_low(&mut self.core.queue.avail_ring, low), + 0x2c => set_high(&mut self.core.queue.avail_ring, low), + common::QUEUE_DEVICE => set_low(&mut self.core.queue.used_ring, low), + 0x34 => set_high(&mut self.core.queue.used_ring, low), + // Read-only fields, and the fields of queues this device does not have. + _ => {} + } + } + + /// A 32-bit word of the device configuration: the capacity in sectors, then the sector size. + fn device_dword(&self, at: u64) -> u32 { + match at { + 0x00 => self.core.capacity_sectors() as u32, + 0x04 => (self.core.capacity_sectors() >> 32) as u32, + 0x14 => SECTOR_SIZE as u32, + _ => 0, + } + } +} + +/// The body of a virtio vendor capability, after its identifier and `next` pointer. The +/// notify structure adds the multiplier that scales a queue's notify offset. +fn structure_body(cfg_type: u8, offset: u64, length: u64, multiplier: Option) -> Vec { + let total = if multiplier.is_some() { 20 } else { 16 }; + let mut body = Vec::with_capacity(total - 2); + body.push(total as u8); // cap_len + body.push(cfg_type); + body.push(BAR as u8); + body.push(0); // id, unused + body.extend_from_slice(&[0, 0]); // padding + body.extend_from_slice(&(offset as u32).to_le_bytes()); + body.extend_from_slice(&(length as u32).to_le_bytes()); + if let Some(multiplier) = multiplier { + body.extend_from_slice(&multiplier.to_le_bytes()); + } + body +} + +fn set_low(field: &mut u64, low: u32) { + *field = (*field & !0xffff_ffff) | u64::from(low); +} + +fn set_high(field: &mut u64, high: u32) { + *field = (*field & 0xffff_ffff) | u64::from(high) << 32; +} + +/// Reads `size` bytes at `offset` from a structure whose words `word` supplies, byte by byte. +fn read_bytes(offset: u64, size: u8, word: impl Fn(u64) -> u32) -> u64 { + (0..u64::from(size)).fold(0, |value, i| { + let at = offset + i; + let byte = (word(at & !3) >> ((at & 3) * 8)) as u8; + value | u64::from(byte) << (8 * i) + }) +} + +impl PciFunction for VirtioBlockPci { + fn config(&self) -> &ConfigSpace { + &self.config + } + + fn config_mut(&mut self) -> &mut ConfigSpace { + &mut self.config + } + + fn bar_read(&mut self, bar: usize, offset: u64, size: u8) -> u64 { + if bar != BAR { + return 0; + } + match offset { + o if o < COMMON_OFFSET + COMMON_LEN => { + read_bytes(o - COMMON_OFFSET, size, |at| self.common_dword(at)) + } + // The notify register is write-only, and the ISR is never raised. + o if (NOTIFY_OFFSET..NOTIFY_OFFSET + NOTIFY_LEN).contains(&o) + || (ISR_OFFSET..ISR_OFFSET + ISR_LEN).contains(&o) => + { + 0 + } + o if (DEVICE_OFFSET..DEVICE_OFFSET + DEVICE_LEN).contains(&o) => { + read_bytes(o - DEVICE_OFFSET, size, |at| self.device_dword(at)) + } + _ => 0, + } + } + + fn bar_write(&mut self, bar: usize, offset: u64, size: u8, value: u64) { + if bar != BAR { + return; + } + match offset { + o if o < COMMON_OFFSET + COMMON_LEN => { + self.common_write(o - COMMON_OFFSET, size, value); + } + // A driver kicks queue 0 with its index. The only queue this device has is 0. + o if (NOTIFY_OFFSET..NOTIFY_OFFSET + NOTIFY_LEN).contains(&o) + && value as u16 == 0 + && self.core.queue.ready => + { + self.core.process(); + } + // The device configuration is read only. + _ => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::devices::virtio_blk::{ + VIRTIO_BLK_S_OK, VIRTIO_BLK_T_IN, VRING_DESC_F_NEXT, VRING_DESC_F_WRITE, + }; + use crate::memory::{GuestMemory, PhysicalMemory, Region}; + + const RAM: u64 = 0x4000_0000; + const DESC: u64 = RAM + 0x1_0000; + const AVAIL: u64 = RAM + 0x1_1000; + const USED: u64 = RAM + 0x1_2000; + const HEADER: u64 = RAM + 0x2_0000; + const DATA: u64 = RAM + 0x2_1000; + const STATUS: u64 = RAM + 0x2_2000; + + fn device(disk: Vec) -> (VirtioBlockPci>, SharedMemory) { + let ram = Region::ram(RAM, alloc::vec![0u8; 1024 * 1024]).unwrap(); + let memory = SharedMemory::new(PhysicalMemory::new(alloc::vec![ram]).unwrap()); + (VirtioBlockPci::new(disk, memory.clone()), memory) + } + + /// The capabilities in list order, as `(structure, offset, length, multiplier)`. The + /// multiplier is read only for the notify structure, which has the longer capability. + fn capabilities(config: &ConfigSpace) -> Vec<(u8, u64, u64, Option)> { + let mut found = Vec::new(); + let mut at = config.read(0x34, 1) as usize; + while at != 0 { + assert_eq!(config.read(at, 1), u64::from(CAP_VENDOR)); + let cfg_type = config.read(at + 3, 1) as u8; + assert_eq!(config.read(at + 4, 1), 0, "every structure is in BAR0"); + let multiplier = (cfg_type == CFG_NOTIFY).then(|| config.read(at + 16, 4)); + found.push(( + cfg_type, + config.read(at + 8, 4), + config.read(at + 12, 4), + multiplier, + )); + at = config.read(at + 1, 1) as usize; + } + found + } + + #[test] + fn capabilities_locate_the_four_structures_in_bar0() { + let (dev, _) = device(alloc::vec![0; 512]); + let config = dev.config(); + assert_ne!(config.read(0x06, 2) & 0x10, 0, "status lists capabilities"); + assert_eq!( + capabilities(config), + [ + (CFG_COMMON, COMMON_OFFSET, COMMON_LEN, None), + ( + CFG_NOTIFY, + NOTIFY_OFFSET, + NOTIFY_LEN, + Some(u64::from(NOTIFY_MULTIPLIER)) + ), + (CFG_ISR, ISR_OFFSET, ISR_LEN, None), + (CFG_DEVICE, DEVICE_OFFSET, DEVICE_LEN, None), + ] + .to_vec() + ); + } + + fn write_descriptor( + memory: &mut SharedMemory, + index: u64, + addr: u64, + len: u32, + flags: u16, + next: u16, + ) { + let mut desc = [0u8; 16]; + desc[0..8].copy_from_slice(&addr.to_le_bytes()); + desc[8..12].copy_from_slice(&len.to_le_bytes()); + desc[12..14].copy_from_slice(&flags.to_le_bytes()); + desc[14..16].copy_from_slice(&next.to_le_bytes()); + memory.write(DESC + index * 16, &desc).unwrap(); + } + + /// Drives the device the way virtio-drivers does: reset, acknowledge, negotiate the + /// features, set FEATURES_OK, and configure and enable queue 0. + fn negotiate(dev: &mut VirtioBlockPci>) { + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1, 0); + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1, 0x3); + dev.bar_write(BAR, COMMON_OFFSET + common::DRIVER_FEATURE_SELECT, 4, 0); + dev.bar_write( + BAR, + COMMON_OFFSET + common::DRIVER_FEATURE, + 4, + VIRTIO_BLK_F_BLK_SIZE | VIRTIO_BLK_F_FLUSH, + ); + dev.bar_write(BAR, COMMON_OFFSET + common::DRIVER_FEATURE_SELECT, 4, 1); + dev.bar_write(BAR, COMMON_OFFSET + common::DRIVER_FEATURE, 4, 1); + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1, 0xb); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_SELECT, 2, 0); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_SIZE, 2, 16); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_DESC, 8, DESC); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_DRIVER, 8, AVAIL); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_DEVICE, 8, USED); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_ENABLE, 2, 1); + } + + #[test] + fn driver_negotiates_features_and_serves_a_read_through_the_pci_transport() { + let mut disk = alloc::vec![0x42u8; 2048]; + disk[512..1024].fill(0x37); // sector 1 is what the request reads + let (mut dev, mut memory) = device(disk); + + // The device offers VERSION_1 in the high word of its features. + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_FEATURE_SELECT, 4, 1); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::DEVICE_FEATURE, 4), + 1 + ); + + negotiate(&mut dev); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1), + 0xb, + "FEATURES_OK is kept when the driver's features are a subset of the device's" + ); + // The configured queue reads back: 8-byte fields come back whole. + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::QUEUE_DESC, 8), + DESC + ); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::QUEUE_ENABLE, 2), + 1 + ); + + // Read sector 1 into DATA: header, data (device-writable), status (device-writable). + let mut header = [0u8; 16]; + header[0..4].copy_from_slice(&VIRTIO_BLK_T_IN.to_le_bytes()); + header[8..16].copy_from_slice(&1u64.to_le_bytes()); + memory.write(HEADER, &header).unwrap(); + write_descriptor(&mut memory, 0, HEADER, 16, VRING_DESC_F_NEXT, 1); + write_descriptor( + &mut memory, + 1, + DATA, + 512, + VRING_DESC_F_WRITE | VRING_DESC_F_NEXT, + 2, + ); + write_descriptor(&mut memory, 2, STATUS, 1, VRING_DESC_F_WRITE, 0); + memory.write(STATUS, &[0xff]).unwrap(); + // Avail: idx 1, ring[0] = descriptor 0. + memory.write(AVAIL + 2, &1u16.to_le_bytes()).unwrap(); + memory.write(AVAIL + 4, &0u16.to_le_bytes()).unwrap(); + + dev.bar_write(BAR, NOTIFY_OFFSET, 2, 0); + + let mut data = [0u8; 512]; + memory.read(DATA, &mut data).unwrap(); + assert_eq!(data, [0x37u8; 512]); + let mut status = [0xffu8]; + memory.read(STATUS, &mut status).unwrap(); + assert_eq!(status[0], VIRTIO_BLK_S_OK); + let mut used_idx = [0u8; 2]; + memory.read(USED + 2, &mut used_idx).unwrap(); + assert_eq!( + u16::from_le_bytes(used_idx), + 1, + "the used ring records the request" + ); + } + + #[test] + fn the_device_configuration_reports_capacity_and_sector_size() { + let (mut dev, _) = device(alloc::vec![0; 2048]); + // Eight-byte capacity is one access, which the transport splits into two words. + assert_eq!(dev.bar_read(BAR, DEVICE_OFFSET, 8), 4); + assert_eq!(dev.bar_read(BAR, DEVICE_OFFSET + 0x14, 4), 512); + assert_eq!(dev.bar_read(BAR, DEVICE_OFFSET, 4), 4); + } + + #[test] + fn resetting_the_status_disables_the_queue() { + let (mut dev, _) = device(alloc::vec![0; 2048]); + negotiate(&mut dev); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::QUEUE_ENABLE, 2), + 1 + ); + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1, 0); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::QUEUE_ENABLE, 2), + 0 + ); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::DRIVER_FEATURE, 4), + 0 + ); + } +} diff --git a/src/emulator/src/system/machine.rs b/src/emulator/src/system/machine.rs index 0e2171f..9c24799 100644 --- a/src/emulator/src/system/machine.rs +++ b/src/emulator/src/system/machine.rs @@ -56,6 +56,17 @@ pub trait DeviceIo { let _ = (cpu_id, address, size, value); false } + /// A read of an address the PCI bus claims: configuration space, or a memory BAR of a + /// function that decodes memory. `None` declines the access. + fn read_pci(&mut self, address: u64, size: u8) -> Option { + let _ = (address, size); + None + } + /// Returns `false` when no PCI function claims the address. + fn write_pci(&mut self, address: u64, size: u8, value: u64) -> bool { + let _ = (address, size, value); + false + } /// Whether the fixed windows below (the QEMU `virt` UART, GIC and virtio) exist on /// this board. A board that maps none of them makes accesses there fault. fn has_fixed_windows(&self) -> bool { @@ -1182,8 +1193,16 @@ impl Machine { .write_soc(self.cpu_id, physical, width, value) .then_some(None) }; + // Only accesses the board's own devices decline reach the PCI bus. + let pci = match (&soc, load) { + (Some(_), _) => None, + (None, true) => devices.read_pci(physical, width).map(Some), + (None, false) => devices.write_pci(physical, width, value).then_some(None), + }; let read = if let Some(read) = soc { read + } else if let Some(read) = pci { + read } else if window(fdt::UART_BASE, crate::devices::pl011::LEN) { if load { Some(devices.read_uart(physical - fdt::UART_BASE, width)) diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index a4fc020..ef2e027 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -16,8 +16,11 @@ pub mod xnu; use crate::aarch64::{Cpu, decode, translate}; use crate::devices::bus::Device; +use crate::devices::pci::PciHost; +use crate::devices::virtio_pci::VirtioBlockPci; use crate::devices::{gicv2::Gicv2, gicv3::Gicv3, pl011::Pl011}; use crate::memory::{GuestMemory, PhysicalMemory, Region, SharedMemory}; +use alloc::boxed::Box; use alloc::sync::Arc; use core::{ sync::atomic::{AtomicBool, Ordering}, @@ -38,6 +41,10 @@ pub const XNU_RAM_SIZE: usize = 1 << 30; /// platform description assumes when no device tree is passed (QEMU's run script uses `-m 4G`). /// The region is zero-filled lazily by the host, so it costs what the guest touches. pub const FIRMWARE_RAM_SIZE: usize = 4 << 30; +/// The distributor's `ITLinesNumber` on a firmware board: 288 INTIDs. `arm-gic` sizes a +/// firmware's interrupt table from it, and an interrupt source beyond the table, such as the +/// physical timer's PPI, cannot be registered when it reads zero. +pub const FIRMWARE_IT_LINES: u8 = 8; #[derive(Debug)] pub enum Error { @@ -285,6 +292,7 @@ struct SystemDevices<'a, V> { gic: &'a mut Intc, virtio: Option<&'a mut V>, soc: Option<&'a mut m1::Soc>, + pci: Option<&'a mut PciHost>, } impl machine::DeviceIo for SystemDevices<'_, V> { @@ -359,6 +367,14 @@ impl machine::DeviceIo for SystemDevices<'_, V> { .as_mut() .is_some_and(|soc| soc.write(cpu_id, address, size, value)) } + fn read_pci(&mut self, address: u64, size: u8) -> Option { + self.pci.as_mut()?.read(address, size) + } + fn write_pci(&mut self, address: u64, size: u8, value: u64) -> bool { + self.pci + .as_mut() + .is_some_and(|pci| pci.write(address, size, value)) + } fn has_fixed_windows(&self) -> bool { self.soc.is_none() } @@ -420,7 +436,7 @@ pub fn boot_with_options( } /// Boot with full system configuration including optional initrd and configurable RAM size. -pub fn boot_system( +pub fn boot_system( image: &[u8], config: SystemConfig, sink: impl FnMut(u8) + Send + 'static, @@ -431,8 +447,8 @@ pub fn boot_system( /// Boot a firmware image the way QEMU's `-bios` does: the image is mapped read-only at /// [`boot::FIRMWARE_BASE`], the core resets there, and no kernel is loaded. The machine /// is the `virt` board with a GICv3, as tinted-boot is run. As under QEMU's `-bios`, `x0` -/// is zero and a device tree sits at RAM base. Firmware boots take no initrd or disk yet. -pub fn boot_firmware( +/// is zero and a device tree sits at RAM base. A disk is a virtio-blk function on PCI; there is no initrd. +pub fn boot_firmware( firmware: &[u8], config: SystemConfig, sink: impl FnMut(u8) + Send + 'static, @@ -447,7 +463,20 @@ enum Payload<'a> { Firmware(&'a [u8]), } -fn boot_machine( +/// The root bus a firmware boot sees: the host bridge at 00:00.0, and the disk, if any, as a +/// virtio-blk function in slot 1. +fn firmware_pci( + disk: Option, + memory: SharedMemory, +) -> PciHost { + let mut host = PciHost::new(); + if let Some(backend) = disk { + host.attach(1, Box::new(VirtioBlockPci::new(backend, memory))); + } + host +} + +fn boot_machine( payload: Payload<'_>, config: SystemConfig, mut sink: impl FnMut(u8) + Send + 'static, @@ -476,9 +505,9 @@ fn boot_machine( ))); } (Payload::Firmware(_), Board::Virt) => { - if config.initrd.is_some() || has_virtio { + if config.initrd.is_some() { return Err(Error::Boot(boot::Error::Unsupported( - "firmware boot has no initrd or virtio-blk yet; the firmware reads its disk over PCI", + "firmware boot has no initrd; the firmware reads its files from a disk", ))); } boot::prepare_firmware(&mut shared_mem, ram_base, ram_size as u64) @@ -546,19 +575,27 @@ fn boot_machine( } else { let serial = Pl011::new(emit).with_line(move |level| serial_line.store(level, Ordering::Release)); - let gic = if is_xnu || matches!(payload, Payload::Firmware(_)) { - Intc::V3(Gicv3::default()) - } else { - Intc::V2(Gicv2::default()) + let gic = match payload { + Payload::Firmware(_) => Intc::V3(Gicv3::default().with_it_lines(FIRMWARE_IT_LINES)), + Payload::Kernel(_) if is_xnu => Intc::V3(Gicv3::default()), + Payload::Kernel(_) => Intc::V2(Gicv2::default()), }; (Some(serial), None, gic) }; let virtio_line = Arc::new(AtomicBool::new(false)); let virtio_notify = virtio_line.clone(); - let mut virtio = config.disk.map(|backend| { - crate::devices::VirtioBlock::new(backend, shared_mem.clone()) - .with_line(move |level| virtio_notify.store(level, Ordering::Release)) - }); + // A firmware boot reaches its disk over PCI, as under QEMU's `-bios`. A kernel boot on the + // virt board keeps the MMIO transport the Linux device tree describes. + let (mut virtio, mut pci) = match (payload, config.disk) { + (Payload::Kernel(_), disk) => ( + disk.map(|backend| { + crate::devices::VirtioBlock::new(backend, shared_mem.clone()) + .with_line(move |level| virtio_notify.store(level, Ordering::Release)) + }), + None, + ), + (Payload::Firmware(_), disk) => (None, Some(firmware_pci(disk, shared_mem.clone()))), + }; let mut machine = machine::Machine::new(shared_mem); if apple { m1::identify(&mut machine.cpu); @@ -610,6 +647,7 @@ fn boot_machine( gic: &mut gic, virtio: virtio.as_mut(), soc: soc.as_mut(), + pci: pci.as_mut(), }; match machine.run_with_devices(&mut dev) { Ok(machine::Exit::Timer) => { @@ -1023,25 +1061,54 @@ mod tests { assert_eq!(report.unmapped_accesses, 0); } + /// A firmware that reads the four bytes of the vendor and device IDs at ECAM slot 1 and + /// writes each to the PL011, then powers off. The ECAM window is 0x40_1000_0000. + fn read_slot_one_ids() -> Vec { + vec![ + 0xd2c00801, // movz x1, #0x40, lsl #32 + 0xf2a20001, // movk x1, #0x1000, lsl #16 + 0xf2900001, // movk x1, #0x8000 (slot 1 = bit 15) + 0xd2a12000, // movz x0, #0x900, lsl #16 (PL011) + 0x39400022, // ldrb w2, [x1] + 0x39000002, // strb w2, [x0] + 0x39400422, // ldrb w2, [x1, #1] + 0x39000002, // strb w2, [x0] + 0x39400822, // ldrb w2, [x1, #2] + 0x39000002, // strb w2, [x0] + 0x39400c22, // ldrb w2, [x1, #3] + 0x39000002, // strb w2, [x0] + 0xd2b08000, // movz x0, #0x8400, lsl #16 + 0xf2800100, // movk x0, #8 (PSCI_SYSTEM_OFF) + 0xd4000002, // hvc #0 + ] + } + + fn firmware_config(disk: Option>) -> SystemConfig> { + SystemConfig { + timeout: Duration::from_secs(5), + cmdline: String::new(), + idle_mode: IdleMode::Paced, + ram_size: 256 << 20, + initrd: None, + disk, + board: Board::Virt, + } + } + #[test] - fn firmware_boot_refuses_a_disk_it_cannot_reach() { - let flash = flash_image(&[0xd4000002]); - let result = boot_firmware( - &flash, - SystemConfig { - timeout: Duration::from_secs(5), - cmdline: String::new(), - idle_mode: IdleMode::Paced, - ram_size: 256 << 20, - initrd: None, - disk: Some(vec![0u8; 512]), - board: Board::Virt, - }, - |_| {}, - ); - assert!(matches!( - result, - Err(Error::Boot(boot::Error::Unsupported(_))) - )); + fn firmware_finds_its_disk_as_a_virtio_function_in_slot_one() { + let flash = flash_image(&read_slot_one_ids()); + let report = boot_firmware(&flash, firmware_config(Some(vec![0; 512])), |_| {}).unwrap(); + assert_eq!(report.reason, StopReason::Shutdown, "{report}"); + // Vendor 0x1af4 and device 0x1042, little-endian. + assert_eq!(report.console, [0xf4, 0x1a, 0x42, 0x10]); + } + + #[test] + fn firmware_without_a_disk_finds_slot_one_empty() { + let flash = flash_image(&read_slot_one_ids()); + let report = boot_firmware(&flash, firmware_config(None), |_| {}).unwrap(); + assert_eq!(report.reason, StopReason::Shutdown, "{report}"); + assert_eq!(report.console, [0xff; 4]); } } From 87983c77d171d102f31827c220d2376bbe2c6f69 Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Fri, 9 Oct 2026 02:53:10 -0700 Subject: [PATCH 10/13] refactor: add volt isa --- .github/workflows/test.yml | 4 +- Cargo.lock | 399 +--- src/emulator/examples/jit_bench.rs | 4 +- src/emulator/src/aarch64/ahead.rs | 35 +- src/emulator/src/aarch64/cache.rs | 6 +- src/emulator/src/aarch64/compile.rs | 1146 +++++----- src/emulator/src/aarch64/cpu.rs | 16 +- src/emulator/src/aarch64/fp.rs | 14 +- src/emulator/src/aarch64/host.rs | 36 + src/emulator/src/aarch64/identification.rs | 2 +- src/emulator/src/aarch64/mod.rs | 2 - src/emulator/src/aarch64/simd_struct.rs | 46 +- src/emulator/src/system/m1.rs | 2 +- src/emulator/src/system/machine.rs | 7 +- src/emulator/src/system/mod.rs | 36 +- src/emulator/src/user/mod.rs | 28 +- src/ir/Cargo.toml | 3 - src/ir/src/attribute.rs | 1 - src/ir/src/bitcode.rs | 848 +++++++- src/ir/src/builder.rs | 67 +- src/ir/src/dialect/attrs.rs | 726 ------- src/ir/src/dialect/func.rs | 1929 ----------------- src/ir/src/dialect/mod.rs | 15 - src/ir/src/dialect/ops.rs | 1674 -------------- src/ir/src/dialect/types.rs | 450 ---- src/ir/src/expand.rs | 252 +-- src/ir/src/expand/tests/interp.rs | 326 +++ src/ir/src/expand/tests/mod.rs | 1 + src/ir/src/expand/tests/tests_matmul.rs | 216 +- src/ir/src/expand/tests/tests_scalar.rs | 506 +++-- src/ir/src/function.rs | 328 ++- src/ir/src/legalize.rs | 1086 +++++++++- src/ir/src/lib.rs | 8 +- src/ir/src/module.rs | 505 +++++ src/ir/src/parser.rs | 836 ++++++- src/ir/src/softfp.rs | 376 +++- src/ir/src/types.rs | 51 +- src/ir/src/verify.rs | 806 ++++++- .../src/aarch64/decode.rs | 233 +- src/target/src/aarch64/disasm.rs | 30 + src/target/src/aarch64/encode.rs | 37 +- src/target/src/aarch64/isel_tests.rs | 1225 ++++++++++- src/target/src/aarch64/jit.rs | 85 + src/target/src/aarch64/link.rs | 382 ++-- src/target/src/aarch64/lower.rs | 451 ++-- src/target/src/aarch64/mod.rs | 2 + src/target/src/aarch64/object.rs | 61 +- src/target/src/aarch64/simd_struct.rs | 51 + src/target/src/lib.rs | 21 - src/target/src/native.rs | 5 +- src/target/src/native/module.rs | 348 ++- src/target/src/regalloc/addrfold.rs | 13 +- src/target/src/regalloc/ir.rs | 5 + src/target/src/regalloc/wimmer.rs | 5 +- src/target/src/x86_64/cases.rs | 676 ++++++ src/target/src/x86_64/codegen_tests.rs | 556 +++++ src/target/src/x86_64/disasm.rs | 29 +- src/target/src/x86_64/encode.rs | 46 +- src/target/src/x86_64/isel.rs | 605 ++++-- src/target/src/x86_64/link.rs | 98 +- src/target/src/x86_64/mod.rs | 4 + src/target/src/x86_64/object.rs | 68 +- src/target/src/x86_64/tests.rs | 525 ++++- 63 files changed, 10861 insertions(+), 7493 deletions(-) delete mode 100644 src/ir/src/dialect/attrs.rs delete mode 100644 src/ir/src/dialect/func.rs delete mode 100644 src/ir/src/dialect/mod.rs delete mode 100644 src/ir/src/dialect/ops.rs delete mode 100644 src/ir/src/dialect/types.rs create mode 100644 src/ir/src/expand/tests/interp.rs create mode 100644 src/ir/src/module.rs rename src/{emulator => target}/src/aarch64/decode.rs (94%) create mode 100644 src/target/src/aarch64/simd_struct.rs create mode 100644 src/target/src/x86_64/cases.rs create mode 100644 src/target/src/x86_64/codegen_tests.rs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93135c9..f355096 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest - name: Setup Mold Linker - uses: rui314/setup-mold@v1 + uses: rui314/setup-mold@staging - name: Run tests with nextest run: mold -run cargo nextest run --no-fail-fast --workspace --all-features lint: @@ -35,7 +35,7 @@ jobs: - name: Cache Cargo dependencies uses: Swatinem/rust-cache@v2 - name: Setup Mold Linker - uses: rui314/setup-mold@v1 + uses: rui314/setup-mold@staging - name: cargo fmt run: mold -run cargo fmt --all --check - name: Run lint diff --git a/Cargo.lock b/Cargo.lock index 7145c02..6c9ce70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,150 +2,18 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - -[[package]] -name = "awint" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a8f298efd080b0d3e1ee1723b5d88ecc0d78805fad347e09b76dbd4dadbe37" -dependencies = [ - "awint_core", - "awint_dag", - "awint_ext", - "awint_macro_internals", - "awint_macros", -] - -[[package]] -name = "awint_core" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24d226d9faa090abec192c8ad1ea74635e6cc94000c72a95ceb60cf691428c3" -dependencies = [ - "awint_internals", - "const_fn", -] - -[[package]] -name = "awint_dag" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5377016f3077b18304da4233a28626ea069047ca84c9bc8c8b81b659f11032" -dependencies = [ - "awint_ext", - "awint_macro_internals", - "awint_macros", - "smallvec", -] - -[[package]] -name = "awint_ext" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ff843cdefdd48fbe9bc11292a902b7921ff37433ed4124fa02136f83186f58" -dependencies = [ - "awint_core", - "const_fn", -] - -[[package]] -name = "awint_internals" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6209d7873685ac95ee41c4b63fe29bff9b2e1106de65d05b106b2b769c178575" -dependencies = [ - "const_fn", -] - -[[package]] -name = "awint_macro_internals" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b519a9cd37bf73e44e29b3779ca90246412b56db4af4e96d959f5082bb8bdcc" -dependencies = [ - "awint_ext", - "proc-macro2", - "triple_arena", -] - -[[package]] -name = "awint_macros" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518a133c2163e8f021f708d7278af27dd6c1bcee96ea672a2d825b13a6931818" -dependencies = [ - "awint_macro_internals", -] - [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -[[package]] -name = "bytemuck" -version = "1.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - [[package]] name = "cfg-if" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" -[[package]] -name = "combine" -version = "4.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" -dependencies = [ - "bytes", - "memchr", -] - -[[package]] -name = "const_fn" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413d67b29ef1021b4d60f4aa1e925ca031751e213832b4b1d588fae623c05c60" - -[[package]] -name = "convert_case" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "crc32fast" version = "1.5.2" @@ -155,18 +23,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "downcast-rs" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - [[package]] name = "equivalent" version = "1.0.2" @@ -191,20 +47,9 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "allocator-api2", - "equivalent", "foldhash", ] -[[package]] -name = "hi_sparse_bitset" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "682097abdf22a902b7b2b5f2824e773691507289873c3ea0abbc5d4c85f05ec0" -dependencies = [ - "wide", -] - [[package]] name = "indexmap" version = "2.14.2" @@ -215,41 +60,12 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "inventory" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" -dependencies = [ - "rustversion", -] - [[package]] name = "libc" version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" -[[package]] -name = "linkme" -version = "0.3.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" -dependencies = [ - "linkme-impl", -] - -[[package]] -name = "linkme-impl" -version = "0.3.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - [[package]] name = "lock_api" version = "0.4.14" @@ -259,12 +75,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - [[package]] name = "memchr" version = "2.8.3" @@ -283,12 +93,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "once_vec" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b61fbf7ec37342ce171d9f5e3b82be74bda6c4e84e3b28338a03d553913e6aa" - [[package]] name = "parking_lot" version = "0.12.5" @@ -312,49 +116,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "pliron" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d59d619aada2eb8f68bacac30cfa373f4e247c7667cdac69370f5eaf0b87178" -dependencies = [ - "awint", - "combine", - "downcast-rs", - "dyn-clone", - "hashbrown", - "hi_sparse_bitset", - "indexmap", - "inventory", - "linkme", - "log", - "once_vec", - "pliron-derive", - "regex", - "rustc-hash", - "rustc_apfloat", - "slotmap", - "smallvec", - "spin", - "thiserror", - "utf8-chars", -] - -[[package]] -name = "pliron-derive" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e9d7f08f5162e44988420d46a12bd9aa776a9c5a0bee5d1555741060c87a73" -dependencies = [ - "combine", - "convert_case", - "indexmap", - "proc-macro2", - "quote", - "rustc-hash", - "syn 3.0.6", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -373,12 +134,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "recasting" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3f51d3a1399f13fa081d97345d0fdf08597fe78558fc8539d0e67412f83b26" - [[package]] name = "redox_syscall" version = "0.5.18" @@ -388,96 +143,18 @@ dependencies = [ "bitflags", ] -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustc_apfloat" -version = "0.2.3+llvm-462a31f5a5ab" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" -dependencies = [ - "bitflags", - "smallvec", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "safe_arch" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42c6efa15875e6ecb39ca61fb0b0c1a40b84fac5a5ffe71eef7d1000c8eb3f5f" -dependencies = [ - "bytemuck", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "slotmap" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" -dependencies = [ - "version_check", -] - [[package]] name = "smallvec" version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" -[[package]] -name = "spin" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10" -dependencies = [ - "lock_api", -] - [[package]] name = "syn" version = "2.0.119" @@ -489,73 +166,12 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "2.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "triple_arena" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee55b64228381514ddc4ad004e631ccf45e594e428d13ce197697ce56ef1c55" -dependencies = [ - "recasting", -] - [[package]] name = "unicode-ident" version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" -[[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "utf8-chars" -version = "3.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92498c67ea3511b37eccff13b573ed871faf0ce9c4056ab032bd01a681f445a0" -dependencies = [ - "arrayvec", -] - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "volt-emulator" version = "0.1.0" @@ -578,9 +194,6 @@ dependencies = [ [[package]] name = "volt-ir" version = "0.1.0" -dependencies = [ - "pliron", -] [[package]] name = "volt-target" @@ -591,16 +204,6 @@ dependencies = [ "volt-ir", ] -[[package]] -name = "wide" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d920ac99c3c8edce110cb8d07dbb324d6d026011dce85b1e9355b70f0adacc4f" -dependencies = [ - "bytemuck", - "safe_arch", -] - [[package]] name = "windows-link" version = "0.2.1" @@ -634,7 +237,7 @@ checksum = "09359adf85eecc7ca2696928137722055b85d380089b159673a161347115adc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] diff --git a/src/emulator/examples/jit_bench.rs b/src/emulator/examples/jit_bench.rs index 9c27285..1a581a7 100644 --- a/src/emulator/examples/jit_bench.rs +++ b/src/emulator/examples/jit_bench.rs @@ -20,12 +20,12 @@ fn block(n: usize, seed: u32) -> Vec { fn main() { dispatch(); { - use volt_emulator::aarch64::decode::decode; + use volt_emulator::aarch64::host::dispatch; let b = block(63, 1); let n = b .chunks_exact(4) .take_while(|c| { - !decode(u32::from_le_bytes((*c).try_into().unwrap())) + !dispatch(u32::from_le_bytes((*c).try_into().unwrap())) .unwrap() .terminates() }) diff --git a/src/emulator/src/aarch64/ahead.rs b/src/emulator/src/aarch64/ahead.rs index 6b34624..d07c453 100644 --- a/src/emulator/src/aarch64/ahead.rs +++ b/src/emulator/src/aarch64/ahead.rs @@ -19,12 +19,13 @@ //! on a worker thread. use super::cache::{MAX_INSNS, SharedBlocks}; -use super::decode::{Instruction, decode}; +use super::host::{Dispatch, dispatch}; use alloc::collections::VecDeque; use alloc::sync::{Arc, Weak}; use std::collections::HashSet; use std::sync::{Condvar, Mutex, MutexGuard, PoisonError}; use std::thread; +use volt_target::aarch64::decode::Instruction; /// Reads guest-physical bytes without any vCPU state, for worker threads. pub trait CodeReader: Send + Sync { @@ -66,10 +67,15 @@ fn lock(state: &Mutex) -> MutexGuard<'_, State> { /// Where control can go after `last`, a block's final instruction at `last_pc`. /// Fixed targets only: an indirect branch or return has none to offer. -fn successors(last: Instruction, last_pc: u64) -> ([u64; 2], usize) { +fn successors(last: Dispatch, last_pc: u64) -> ([u64; 2], usize) { let next = last_pc.wrapping_add(4); let at = |offset: i64| last_pc.wrapping_add(offset as u64); - match last { + let instruction = match last { + Dispatch::Guest(instruction) => instruction, + // A host word falls through, as a block end that does not branch does. + Dispatch::Host(_) => return ([next, 0], 1), + }; + match instruction { Instruction::B(offset) => ([at(offset), 0], 1), Instruction::Call(call) if call.link => ([at(call.target), next], 2), Instruction::Call(call) => ([at(call.target), 0], 1), @@ -113,7 +119,7 @@ impl Ahead { /// Offer the blocks that can follow `last` (the final instruction of the /// block just built, at `last_pc`). `physical` is the physical address of /// `last_pc`; only successors on its page are taken. - pub fn offer_successors(&self, last: Instruction, last_pc: u64, physical: u64) { + pub fn offer_successors(&self, last: Dispatch, last_pc: u64, physical: u64) { let (targets, count) = successors(last, last_pc); self.offer(&targets[..count], last_pc, physical, 0); } @@ -182,7 +188,7 @@ impl Ahead { return; } let word = u32::from_le_bytes(page[count..count + 4].try_into().unwrap()); - let Ok(instruction) = decode(word) else { + let Ok(instruction) = dispatch(word) else { return; }; last = Some((instruction, job.pc + count as u64)); @@ -259,7 +265,7 @@ mod tests { #[test] fn fixed_targets_follow_each_kind_of_block_end() { - use crate::aarch64::decode::{Call, Test, Width}; + use volt_target::aarch64::decode::{Call, Test, Width}; let at = 0x1000; let test = Instruction::TestBranch(Test { rt: 0, @@ -275,19 +281,22 @@ mod tests { (call(false), vec![0x1040]), (test, vec![0x0ff8, 0x1004]), ( - Instruction::Indirect(crate::aarch64::decode::Indirect { rn: 1, link: false }), + Instruction::Indirect(volt_target::aarch64::decode::Indirect { + rn: 1, + link: false, + }), vec![], ), ( - Instruction::Indirect(crate::aarch64::decode::Indirect { rn: 1, link: true }), + Instruction::Indirect(volt_target::aarch64::decode::Indirect { rn: 1, link: true }), vec![0x1004], ), (Instruction::Eret, vec![]), - (Instruction::Svc, vec![0x1004]), + (Instruction::Svc(0), vec![0x1004]), (Instruction::Nop, vec![0x1004]), ]; for (last, expected) in cases { - let (targets, n) = successors(last, at); + let (targets, n) = successors(Dispatch::Guest(last), at); assert_eq!(&targets[..n], expected.as_slice(), "{last:?}"); } } @@ -304,14 +313,14 @@ mod tests { ]); let shared = Arc::new(SharedBlocks::new(false)); let ahead = Ahead::start(&shared, memory, 2); - ahead.offer_successors(Instruction::B(0x10), 0x1004, 0x1004); + ahead.offer_successors(Dispatch::Guest(Instruction::B(0x10)), 0x1004, 0x1004); wait_for(&shared, 2); // The block after the branch, and the one after its `svc`: nothing more, // even though the last one branches to itself. thread::sleep(Duration::from_millis(50)); assert_eq!(shared.compiled(), 2); // Offering again builds nothing new. - ahead.offer_successors(Instruction::B(0x10), 0x1004, 0x1004); + ahead.offer_successors(Dispatch::Guest(Instruction::B(0x10)), 0x1004, 0x1004); thread::sleep(Duration::from_millis(50)); assert_eq!(shared.compiled(), 2); // The vCPU's own lookup of the same bytes finds the block already built. @@ -328,7 +337,7 @@ mod tests { let shared = Arc::new(SharedBlocks::new(false)); let ahead = Ahead::start(&shared, memory, 1); // A branch from page 1 to page 2 must not be read on this page's translation. - ahead.offer_successors(Instruction::B(0x1004), 0x1ffc, 0x1ffc); + ahead.offer_successors(Dispatch::Guest(Instruction::B(0x1004)), 0x1ffc, 0x1ffc); thread::sleep(Duration::from_millis(50)); assert_eq!(shared.compiled(), 0); ahead.shutdown(); diff --git a/src/emulator/src/aarch64/cache.rs b/src/emulator/src/aarch64/cache.rs index 604a25a..65dd573 100644 --- a/src/emulator/src/aarch64/cache.rs +++ b/src/emulator/src/aarch64/cache.rs @@ -2,7 +2,7 @@ use super::{ ahead::{Ahead, CodeReader}, compile::{Block, Error, compile_with}, cpu::Cpu, - decode::decode, + host::dispatch, }; use core::hash::{Hash, Hasher}; use core::sync::atomic::{AtomicUsize, Ordering}; @@ -356,7 +356,7 @@ impl Cache { } count += 4; let word = u32::from_le_bytes(room.try_into().unwrap()); - let instruction = match decode(word) { + let instruction = match dispatch(word) { Ok(insn) => insn, // Run the decoded prefix first: the undecodable word then heads the // next block, where it is reported at its own address. @@ -366,7 +366,7 @@ impl Cache { } Err(_) => { // Architecturally undefined words are the guest's to handle. - if !super::decode::is_undefined(word) { + if !volt_target::aarch64::decode::is_undefined(word) { eprintln!("cache decode failure at pc {at:x}: word {word:08x}"); } return Err(Error::Decode { word, pc: at }); diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index 757e75c..c9300c9 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -1,6 +1,6 @@ use super::{ cpu::{Cpu, DTLB_ENTRIES, Exclusive as CpuExclusive, System as CpuSystem, Trap}, - decode::*, + host::{Dispatch, dispatch}, }; use crate::memory::MemoryError; use core::{ @@ -12,6 +12,7 @@ use volt_ir::{ function::{self as ir, BinOp as B, CmpOp as C, Function, Opcode, Value}, types::{IntDesc, Type, TypeKind}, }; +use volt_target::aarch64::decode::*; use volt_target::native; #[derive(Debug)] @@ -80,9 +81,9 @@ struct Lower { /// Block currently being emitted into. Memory fast paths switch it. block: Cell, cpu: Value, - u64: Type, - u32: Type, - u8: Type, + i64: Type, + i32: Type, + i8: Type, boolean: Type, ptr: Type, /// Guest instructions started but not yet added to `cntvct_el0`. The @@ -95,18 +96,9 @@ impl Lower { let mut f = Function::new(); // A typical block is a few blocks and about a hundred instructions. f.reserve(8, 128, 128); - let u64 = f.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits: 64, - })); - let u32 = f.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits: 32, - })); - let u8 = f.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits: 8, - })); + let i64 = f.types.intern(TypeKind::Int(IntDesc { bits: 64 })); + let i32 = f.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let i8 = f.types.intern(TypeKind::Int(IntDesc { bits: 8 })); let boolean = f.types.intern(TypeKind::Bool); let ptr = f.types.ptr_global(); let block = f.append_block(); @@ -115,9 +107,9 @@ impl Lower { f: RefCell::new(f), block: Cell::new(block), cpu, - u64, - u32, - u8, + i64, + i32, + i8, boolean, ptr, pending: Cell::new(0), @@ -146,12 +138,23 @@ impl Lower { ) } fn cv(&self, t: Type, v: Value) -> Value { - let same = { self.f.borrow().value_type(v) == t }; - if same { - v - } else { - self.emit(t, Opcode::Convert(ir::Convert { value: v })) + let from = self.f.borrow().value_type(v); + if from == t { + return v; } + let kind = { + let f = self.f.borrow(); + let bits = |ty| match f.types.type_kind(ty) { + TypeKind::Int(d) => d.bits, + _ => 1, + }; + if bits(t) > bits(from) { + ir::ConvertKind::Zext + } else { + ir::ConvertKind::Trunc + } + }; + self.emit(t, Opcode::Convert(ir::Convert { value: v, kind })) } fn cmp(&self, op: C, a: Value, b: Value) -> Value { self.emit( @@ -177,7 +180,7 @@ impl Lower { t, Opcode::Load(ir::Load { ptr: p, - volatile: false, + mem: ir::MemFlags::default(), }), ) } @@ -191,7 +194,7 @@ impl Lower { self.store_ptr(self.addr(at), v) } fn byte(&self, at: usize, v: u64) { - self.store(at, self.k(self.u8, v)) + self.store(at, self.k(self.i8, v)) } fn reg_at(r: u8) -> usize { if r == 31 { @@ -204,7 +207,7 @@ impl Lower { if zero && r == 31 { self.k(t, 0) } else { - self.cv(t, self.load(self.u64, Self::reg_at(r))) + self.cv(t, self.load(self.i64, Self::reg_at(r))) } } fn put(&self, r: u8, v: Value) { @@ -213,20 +216,14 @@ impl Lower { } } fn put_sp(&self, r: u8, v: Value) { - self.store(Self::reg_at(r), self.cv(self.u64, v)) + self.store(Self::reg_at(r), self.cv(self.i64, v)) } fn ty(&self, w: Width) -> Type { - if w == Width::X64 { self.u64 } else { self.u32 } - } - fn signed(&self, w: Width) -> Type { - self.f.borrow_mut().types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: w as u16, - })) + if w == Width::X64 { self.i64 } else { self.i32 } } fn add_counter(&self, n: u64) { let at = offset_of!(Cpu, system) + offset_of!(CpuSystem, cntvct_el0); - self.store(at, self.imm(self.u64, B::Add, self.load(self.u64, at), n)); + self.store(at, self.imm(self.i64, B::Add, self.load(self.i64, at), n)); } /// Make `cntvct_el0` in memory current on the path being emitted. fn flush_counter(&self) { @@ -252,12 +249,12 @@ impl Lower { fn flag(&self, flags: Value, bit: u32) -> Value { self.cmp( C::Ne, - self.imm(self.u32, B::BitAnd, flags, 1 << bit), - self.k(self.u32, 0), + self.imm(self.i32, B::BitAnd, flags, 1 << bit), + self.k(self.i32, 0), ) } fn condition(&self, c: Condition) -> Value { - let f = self.load(self.u32, offset_of!(Cpu, flags)); + let f = self.load(self.i32, offset_of!(Cpu, flags)); let n = self.flag(f, 31); let z = self.flag(f, 30); let carry = self.flag(f, 29); @@ -284,13 +281,13 @@ impl Lower { } } fn pack(&self, n: Value, z: Value, c: Value, v: Value) -> Value { - let mut out = self.k(self.u32, 0); + let mut out = self.k(self.i32, 0); for (bit, shift) in [(n, 31), (z, 30), (c, 29), (v, 28)] { out = self.bin( - self.u32, + self.i32, B::BitOr, out, - self.imm(self.u32, B::Shl, self.cv(self.u32, bit), shift), + self.imm(self.i32, B::Shl, self.cv(self.i32, bit), shift), ); } out @@ -314,16 +311,14 @@ impl Lower { let z = self.cmp(C::Eq, r, zero); let c = carry.unwrap_or_else(|| { if op == ArithOp::Sub { - self.cmp(C::Ge, a, b) + self.cmp(C::Uge, a, b) } else { - self.cmp(C::Lt, r, a) + self.cmp(C::Ult, r, a) } }); - let s = self.signed(w); - let sz = self.k(s, 0); - let an = self.cmp(C::Lt, self.cv(s, a), sz); - let bn = self.cmp(C::Lt, self.cv(s, b), sz); - let rn = self.cmp(C::Lt, self.cv(s, r), sz); + let an = self.cmp(C::Slt, a, zero); + let bn = self.cmp(C::Slt, b, zero); + let rn = self.cmp(C::Slt, r, zero); let diff = self.bin(self.boolean, B::BitXor, an, bn); let flip = self.bin(self.boolean, B::BitXor, an, rn); let v = self.bin( @@ -358,10 +353,7 @@ impl Lower { match s { Shift::Lsl => self.imm(t, B::Shl, v, n), Shift::Lsr => self.imm(t, B::Shr, v, n), - Shift::Asr => { - let signed = self.signed(w); - self.cv(t, self.imm(signed, B::Shr, self.cv(signed, v), n)) - } + Shift::Asr => self.imm(t, B::Sar, v, n), Shift::Ror => self.bin( t, B::BitOr, @@ -373,62 +365,62 @@ impl Lower { fn extend(&self, v: Value, e: Extend) -> Value { match e { Extend::Uxtx | Extend::Sxtx => v, - Extend::Uxtb => self.imm(self.u64, B::BitAnd, v, 0xff), - Extend::Uxth => self.imm(self.u64, B::BitAnd, v, 0xffff), - Extend::Uxtw => self.imm(self.u64, B::BitAnd, v, 0xffffffff), + Extend::Uxtb => self.imm(self.i64, B::BitAnd, v, 0xff), + Extend::Uxth => self.imm(self.i64, B::BitAnd, v, 0xffff), + Extend::Uxtw => self.imm(self.i64, B::BitAnd, v, 0xffffffff), Extend::Sxtb | Extend::Sxth | Extend::Sxtw => { let n = match e { Extend::Sxtb => 56, Extend::Sxth => 48, _ => 32, }; - let raised = self.imm(self.u64, B::Shl, v, n); + let raised = self.imm(self.i64, B::Shl, v, n); self.shift(Width::X64, raised, Shift::Asr, n) } } } fn address(&self, rn: u8, a: Addressing) -> Value { - let base = self.reg(self.u64, rn, false); + let base = self.reg(self.i64, rn, false); match a { - Addressing::Offset(n) => self.imm(self.u64, B::Add, base, n as u64), + Addressing::Offset(n) => self.imm(self.i64, B::Add, base, n as u64), Addressing::PreIndex(n) => { - let a = self.imm(self.u64, B::Add, base, n as u64); + let a = self.imm(self.i64, B::Add, base, n as u64); self.put_sp(rn, a); a } Addressing::PostIndex(n) => { self.store( offset_of!(Cpu, writeback_value), - self.imm(self.u64, B::Add, base, n as u64), + self.imm(self.i64, B::Add, base, n as u64), ); self.byte(offset_of!(Cpu, writeback_dest), rn as u64); self.byte(offset_of!(Cpu, writeback), 1); base } Addressing::Register { rm, extend, amount } => { - let v = self.extend(self.reg(self.u64, rm, true), extend); + let v = self.extend(self.reg(self.i64, rm, true), extend); self.bin( - self.u64, + self.i64, B::Add, base, - self.imm(self.u64, B::Shl, v, amount as u64), + self.imm(self.i64, B::Shl, v, amount as u64), ) } } } fn branch(&self, pc: u64, offset: i64, c: Value) { self.ret(self.sel( - self.u64, + self.i64, c, - self.k(self.u64, pc.wrapping_add(offset as u64)), - self.k(self.u64, pc.wrapping_add(4)), + self.k(self.i64, pc.wrapping_add(offset as u64)), + self.k(self.i64, pc.wrapping_add(4)), )) } - fn uint(&self, bits: u16) -> Type { - self.f.borrow_mut().types.intern(TypeKind::Int(IntDesc { - signed: false, - bits, - })) + fn int(&self, bits: u16) -> Type { + self.f + .borrow_mut() + .types + .intern(TypeKind::Int(IntDesc { bits })) } fn new_block(&self) -> ir::Block { self.f.borrow_mut().append_block() @@ -448,12 +440,12 @@ impl Lower { fn address_split(&self, rn: u8, a: Addressing) -> (Value, Option<(u8, Value)>) { match a { Addressing::PostIndex(n) => { - let base = self.reg(self.u64, rn, false); - (base, Some((rn, self.imm(self.u64, B::Add, base, n as u64)))) + let base = self.reg(self.i64, rn, false); + (base, Some((rn, self.imm(self.i64, B::Add, base, n as u64)))) } Addressing::PreIndex(n) => { - let base = self.reg(self.u64, rn, false); - let address = self.imm(self.u64, B::Add, base, n as u64); + let base = self.reg(self.i64, rn, false); + let address = self.imm(self.i64, B::Add, base, n as u64); (address, Some((rn, address))) } other => (self.address(rn, other), None), @@ -468,39 +460,39 @@ impl Lower { let n = size as u64; // Entry offset = ((address >> 12) & (ENTRIES - 1)) * 32 = (address >> 7) & mask. let index = self.imm( - self.u64, + self.i64, B::BitAnd, - self.imm(self.u64, B::Shr, address, 7), + self.imm(self.i64, B::Shr, address, 7), (DTLB_ENTRIES as u64 - 1) << 5, ); let entry = self.bin(self.ptr, B::Add, self.cpu, index); // Constant displacements fold into the load addressing. let field = |at: usize| { self.load_ptr( - self.u64, + self.i64, self.imm(self.ptr, B::Add, entry, (offset_of!(Cpu, dtlb) + at) as u64), ) }; let tag = field(if write { 8 } else { 0 }); let addend = field(16); let host = self.bin(self.ptr, B::Add, address, addend); - let want = self.imm(self.u64, B::BitAnd, address, !0xfffu64 | (n - 1)); + let want = self.imm(self.i64, B::BitAnd, address, !0xfffu64 | (n - 1)); // Last, so the backend can fuse it into the branch that consumes it. let hit = self.cmp(C::Eq, tag, want); (hit, host) } fn fast_load(&self, host: Value, size: Size, rt: u8, signed: SignExtend) { let bits = size as u16 * 8; - let raw = self.load_ptr(self.uint(bits), host); + let raw = self.load_ptr(self.int(bits), host); if rt == 31 { return; } - let mut v = self.cv(self.u64, raw); + let mut v = self.cv(self.i64, raw); if signed != SignExtend::None && bits < 64 { let n = 64 - bits as u64; - let extended = self.shift(Width::X64, self.imm(self.u64, B::Shl, v, n), Shift::Asr, n); + let extended = self.shift(Width::X64, self.imm(self.i64, B::Shl, v, n), Shift::Asr, n); v = if signed == SignExtend::To32 { - self.imm(self.u64, B::BitAnd, extended, 0xffff_ffff) + self.imm(self.i64, B::BitAnd, extended, 0xffff_ffff) } else { extended }; @@ -508,8 +500,8 @@ impl Lower { self.put(rt, v); } fn fast_store(&self, host: Value, size: Size, rt: u8) { - let t = self.uint(size as u16 * 8); - self.store_ptr(host, self.cv(t, self.reg(self.u64, rt, true))); + let t = self.int(size as u16 * 8); + self.store_ptr(host, self.cv(t, self.reg(self.i64, rt, true))); } /// Emit a load or store with an inline fast path. On a data-TLB hit the /// access happens in place and emission continues in a fresh block; on a @@ -561,9 +553,9 @@ impl Lower { // `address` is aligned to `n`, so the second access leaves the page // exactly when it starts on a page boundary. self.block.set(check_block); - let second_at = self.imm(self.u64, B::Add, address, n); - let offset = self.imm(self.u64, B::BitAnd, second_at, 0xfff); - let same_page = self.cmp(C::Ne, offset, self.k(self.u64, 0)); + let second_at = self.imm(self.i64, B::Add, address, n); + let offset = self.imm(self.i64, B::BitAnd, second_at, 0xfff); + let same_page = self.cmp(C::Ne, offset, self.k(self.i64, 0)); self.branch_if(same_page, fast_block, slow_block); self.block.set(slow_block); slow(self); @@ -582,7 +574,7 @@ impl Lower { } fn trap(&self, pc: u64, t: Trap) { self.byte(offset_of!(Cpu, trap), t as u64); - self.ret(self.k(self.u64, pc)) + self.ret(self.k(self.i64, pc)) } /// Slow path for a whole SIMD vector or vector-pair access: describe the @@ -604,7 +596,7 @@ impl Lower { self.byte(offset_of!(Cpu, dest), rt as u64); self.byte(offset_of!(Cpu, vector_dest), 1); if let Some(rt2) = rt2 { - let second = self.imm(self.u64, B::Add, address, if bytes == 16 { 16 } else { 8 }); + let second = self.imm(self.i64, B::Add, address, if bytes == 16 { 16 } else { 8 }); self.store(offset_of!(Cpu, second_address), second); self.byte(offset_of!(Cpu, second_width), bytes); self.byte(offset_of!(Cpu, second_dest), rt2 as u64); @@ -653,8 +645,15 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< for chunk in bytes.chunks_exact(4) { l.pending.set(l.pending.get() + 1); let word = u32::from_le_bytes(chunk.try_into().unwrap()); - let instruction = match decode(word) { - Ok(insn) => insn, + let instruction = match dispatch(word) { + Ok(Dispatch::Guest(insn)) => insn, + Ok(Dispatch::Host(host_word)) => { + // The host runs this word (see `host::run`), and it ends the block. + l.store(offset_of!(Cpu, value), l.k(l.i64, host_word as u64)); + l.trap(pc.wrapping_add(4), Trap::Host); + ended = true; + break; + } Err(_) => { eprintln!("compile decode failure at pc {pc:x}: word {word:08x}, full block:"); for (i, c) in bytes.chunks_exact(4).enumerate() { @@ -727,8 +726,8 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< Instruction::ArithExt(a) => { let t = l.ty(a.width); let lhs = l.reg(t, a.rn, false); - let extended = l.extend(l.reg(l.u64, a.rm, true), a.extend); - let rhs = l.cv(t, l.imm(l.u64, B::Shl, extended, a.amount as u64)); + let extended = l.extend(l.reg(l.i64, a.rm, true), a.extend); + let rhs = l.cv(t, l.imm(l.i64, B::Shl, extended, a.amount as u64)); let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); if a.flags { l.put(a.rd, r); @@ -777,9 +776,9 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let carry = l.cv( t, l.imm( - l.u32, + l.i32, B::BitAnd, - l.imm(l.u32, B::Shr, l.load(l.u32, offset_of!(Cpu, flags)), 29), + l.imm(l.i32, B::Shr, l.load(l.i32, offset_of!(Cpu, flags)), 29), 1, ), ); @@ -792,8 +791,8 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let out = l.bin( l.boolean, B::BitOr, - l.cmp(C::Lt, partial, lhs), - l.cmp(C::Lt, r, partial), + l.cmp(C::Ult, partial, lhs), + l.cmp(C::Ult, r, partial), ); l.store( offset_of!(Cpu, flags), @@ -813,10 +812,10 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< l.store( offset_of!(Cpu, flags), l.sel( - l.u32, + l.i32, l.condition(a.cond), flags, - l.k(l.u32, (a.nzcv as u64) << 28), + l.k(l.i32, (a.nzcv as u64) << 28), ), ); } @@ -856,12 +855,12 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< Instruction::MulLong(a) => { let e = if a.signed { Extend::Sxtw } else { Extend::Uxtw }; let p = l.bin( - l.u64, + l.i64, B::Mul, - l.extend(l.reg(l.u64, a.rn, true), e), - l.extend(l.reg(l.u64, a.rm, true), e), + l.extend(l.reg(l.i64, a.rn, true), e), + l.extend(l.reg(l.i64, a.rm, true), e), ); - let r = l.bin(l.u64, arithmetic_op(a.op), l.reg(l.u64, a.ra, true), p); + let r = l.bin(l.i64, arithmetic_op(a.op), l.reg(l.i64, a.ra, true), p); if a.rd != 31 { l.put(a.rd, r) } @@ -869,7 +868,7 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< Instruction::MulHigh(a) => lower_high(&l, a), Instruction::Adr(a) => { let here = if a.page { pc & 0xfffffffffffff000 } else { pc }; - l.put(a.rd, l.k(l.u64, here.wrapping_add(a.offset as u64))); + l.put(a.rd, l.k(l.i64, here.wrapping_add(a.offset as u64))); } Instruction::Csel(a) => { let t = l.ty(a.width); @@ -901,17 +900,17 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< ); } Instruction::BCond(a) => l.branch(pc, a.offset, l.condition(a.cond)), - Instruction::B(offset) => l.ret(l.k(l.u64, pc.wrapping_add(offset as u64))), + Instruction::B(offset) => l.ret(l.k(l.i64, pc.wrapping_add(offset as u64))), Instruction::Call(a) => { if a.link { - l.put(30, l.k(l.u64, pc.wrapping_add(4))) + l.put(30, l.k(l.i64, pc.wrapping_add(4))) } - l.ret(l.k(l.u64, pc.wrapping_add(a.target as u64))); + l.ret(l.k(l.i64, pc.wrapping_add(a.target as u64))); } Instruction::Indirect(a) => { - let target = l.reg(l.u64, a.rn, false); + let target = l.reg(l.i64, a.rn, false); if a.link { - l.put(30, l.k(l.u64, pc.wrapping_add(4))) + l.put(30, l.k(l.i64, pc.wrapping_add(4))) } l.ret(target); } @@ -940,7 +939,7 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } } Instruction::Literal(a) => { - let address = l.k(l.u64, pc.wrapping_add(a.offset as u64)); + let address = l.k(l.i64, pc.wrapping_add(a.offset as u64)); if inline_memory { l.memory_access( address, @@ -967,13 +966,13 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } } Instruction::SimdLiteral(a) => { - let address = l.k(l.u64, pc.wrapping_add(a.offset as u64)); + let address = l.k(l.i64, pc.wrapping_add(a.offset as u64)); l.simd_trap_access(address, a.bytes as u64, a.rt, None, None, pc, Trap::Load); } Instruction::Exclusive(a) => { setup_memory( &l, - l.reg(l.u64, a.rn, false), + l.reg(l.i64, a.rn, false), a.size, a.rt, SignExtend::None, @@ -1010,7 +1009,7 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< AtomicKind::StorePair => ([31, 31], &[a.rt, a.rt2]), _ => ([a.rt, 31], &[a.rs]), }; - l.store(offset_of!(Cpu, address), l.reg(l.u64, a.rn, false)); + l.store(offset_of!(Cpu, address), l.reg(l.i64, a.rn, false)); l.byte(offset_of!(Cpu, width), a.size as u64); l.byte(offset_of!(Cpu, dest), dests[0] as u64); l.byte(offset_of!(Cpu, atomic_kind), a.kind as u64); @@ -1021,7 +1020,7 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< // Register numbers past 30 read as XZR. l.store( offset_of!(Cpu, atomic_operands) + slot * 8, - l.reg(l.u64, reg, true), + l.reg(l.i64, reg, true), ); } if a.kind == AtomicKind::StorePair { @@ -1030,10 +1029,10 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< l.trap(pc.wrapping_add(4), Trap::Atomic); } Instruction::SimdStruct(a) => { - use super::simd_struct::StructDesc; + use volt_target::aarch64::simd_struct::StructDesc; // Element-wise accesses with lane merging need the machine slow path. let d = a.desc; - let address = l.reg(l.u64, a.rn, false); + let address = l.reg(l.i64, a.rn, false); l.store(offset_of!(Cpu, address), address); for (field, value) in [ (offset_of!(StructDesc, store), u64::from(d.store)), @@ -1052,13 +1051,13 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } let step = match a.post { StructPost::None => None, - StructPost::Immediate => Some(l.k(l.u64, d.bytes())), - StructPost::Register(rm) => Some(l.reg(l.u64, rm, true)), + StructPost::Immediate => Some(l.k(l.i64, d.bytes())), + StructPost::Register(rm) => Some(l.reg(l.i64, rm, true)), }; if let Some(step) = step { l.store( offset_of!(Cpu, writeback_value), - l.bin(l.u64, B::Add, address, step), + l.bin(l.i64, B::Add, address, step), ); l.byte(offset_of!(Cpu, writeback_dest), a.rn as u64); l.byte(offset_of!(Cpu, writeback), 1); @@ -1066,18 +1065,13 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< l.trap(pc.wrapping_add(4), Trap::SimdStruct); } Instruction::AddressTranslate(a) => { - l.store(offset_of!(Cpu, address), l.reg(l.u64, a.rt, true)); + l.store(offset_of!(Cpu, address), l.reg(l.i64, a.rt, true)); l.store( offset_of!(Cpu, value), - l.k(l.u64, u64::from(a.write) | u64::from(a.user) << 1), + l.k(l.i64, u64::from(a.write) | u64::from(a.user) << 1), ); l.trap(pc.wrapping_add(4), Trap::AddressTranslate); } - Instruction::Host(word) => { - // The host runs this word: see `host::run`. - l.store(offset_of!(Cpu, value), l.k(l.u64, word as u64)); - l.trap(pc.wrapping_add(4), Trap::Host); - } Instruction::Pair(a) => { let trap = if a.op == MemoryOp::Store { Trap::Store @@ -1101,14 +1095,14 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< l.trap(pc.wrapping_add(4), trap); } } - Instruction::Clrex => l.byte(offset_of!(Cpu, monitor_valid), 0), + Instruction::Clrex(_) => l.byte(offset_of!(Cpu, monitor_valid), 0), Instruction::Sev | Instruction::Sevl => l.byte(offset_of!(Cpu, event_set), 1), Instruction::SimdImm(a) => { let at = offset_of!(Cpu, v) + a.rd as usize * 16; match a.combine { ImmCombine::Set => { - l.store(at, l.k(l.u64, a.low)); - l.store(at + 8, l.k(l.u64, a.high)); + l.store(at, l.k(l.i64, a.low)); + l.store(at + 8, l.k(l.i64, a.high)); } ImmCombine::Or | ImmCombine::AndNot => { // A 64-bit form (`q == false`) also clears the upper half. @@ -1116,13 +1110,13 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< ImmCombine::Or => (B::BitOr, a.low), _ => (B::BitAnd, !a.low), }; - l.store(at, l.imm(l.u64, op, l.load(l.u64, at), imm)); + l.store(at, l.imm(l.i64, op, l.load(l.i64, at), imm)); l.store( at + 8, if a.q { - l.imm(l.u64, op, l.load(l.u64, at + 8), imm) + l.imm(l.i64, op, l.load(l.i64, at + 8), imm) } else { - l.k(l.u64, 0) + l.k(l.i64, 0) }, ); } @@ -1131,8 +1125,8 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< Instruction::SimdMove { rd, rn } => { let at_d = offset_of!(Cpu, v) + rd as usize * 16; let at_n = offset_of!(Cpu, v) + rn as usize * 16; - let lo = l.load(l.u64, at_n); - let hi = l.load(l.u64, at_n + 8); + let lo = l.load(l.i64, at_n); + let hi = l.load(l.i64, at_n + 8); l.store(at_d, lo); l.store(at_d + 8, hi); } @@ -1148,17 +1142,17 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let per_dw = (64 / ebits) as usize; let word = a.index as usize / per_dw; let shift = (a.index as usize % per_dw) as u64 * ebits; - let old = l.load(l.u64, at_d + word * 8); - let cleared = l.bin(l.u64, B::BitAnd, old, l.k(l.u64, !(emask << shift))); + let old = l.load(l.i64, at_d + word * 8); + let cleared = l.bin(l.i64, B::BitAnd, old, l.k(l.i64, !(emask << shift))); let value = l.bin( - l.u64, + l.i64, B::BitAnd, - l.reg(l.u64, a.rn, true), - l.k(l.u64, emask), + l.reg(l.i64, a.rn, true), + l.k(l.i64, emask), ); l.store( at_d + word * 8, - l.bin(l.u64, B::BitOr, cleared, l.imm(l.u64, B::Shl, value, shift)), + l.bin(l.i64, B::BitOr, cleared, l.imm(l.i64, B::Shl, value, shift)), ); } Instruction::SimdInsertElement(a) => { @@ -1181,22 +1175,22 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< a.dst as usize / per_dw, (a.dst as usize % per_dw) as u64 * ebits, ); - let source = l.load(l.u64, at_n + src_word * 8); + let source = l.load(l.i64, at_n + src_word * 8); let value = l.imm( - l.u64, + l.i64, B::BitAnd, - l.imm(l.u64, B::Shr, source, src_shift), + l.imm(l.i64, B::Shr, source, src_shift), emask, ); - let old = l.load(l.u64, at_d + dst_word * 8); - let cleared = l.imm(l.u64, B::BitAnd, old, !(emask << dst_shift)); + let old = l.load(l.i64, at_d + dst_word * 8); + let cleared = l.imm(l.i64, B::BitAnd, old, !(emask << dst_shift)); l.store( at_d + dst_word * 8, l.bin( - l.u64, + l.i64, B::BitOr, cleared, - l.imm(l.u64, B::Shl, value, dst_shift), + l.imm(l.i64, B::Shl, value, dst_shift), ), ); } @@ -1212,8 +1206,8 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let per_dw = (64 / ebits) as usize; let word = a.index as usize / per_dw; let shift = (a.index as usize % per_dw) as u64 * ebits; - let source = l.load(l.u64, at_n + word * 8); - let value = l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, source, shift), emask); + let source = l.load(l.i64, at_n + word * 8); + let value = l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, source, shift), emask); l.put(a.rd, value); } Instruction::SimdDupElement(a) => { @@ -1224,8 +1218,8 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< (1u64 << ebits) - 1 }; let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); + let vn_lo = l.load(l.i64, at_n); + let vn_hi = l.load(l.i64, at_n + 8); // Locate the requested element across the 128-bit source. let lane = a.index as usize; let lanes_per_dw = (64 / ebits) as usize; @@ -1235,52 +1229,52 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< vn_hi }; let shift = (lane % lanes_per_dw) as u64 * ebits; - let element = l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, src, shift), emask); + let element = l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, src, shift), emask); // Replicate the element across the destination lanes. - let mut acc = l.k(l.u64, 0); + let mut acc = l.k(l.i64, 0); let lanes = (64 / ebits) as usize; for lane in 0..lanes { acc = l.bin( - l.u64, + l.i64, B::BitOr, acc, - l.imm(l.u64, B::Shl, element, lane as u64 * ebits), + l.imm(l.i64, B::Shl, element, lane as u64 * ebits), ); } let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; l.store(at_d, acc); - l.store(at_d + 8, if a.q { acc } else { l.k(l.u64, 0) }); + l.store(at_d + 8, if a.q { acc } else { l.k(l.i64, 0) }); } Instruction::SimdDup { rd, rn, esize, q } => { - let r = l.reg(l.u64, rn, true); + let r = l.reg(l.i64, rn, true); let val64 = match esize { 0 => { - let b = l.imm(l.u64, B::BitAnd, r, 0xff); - l.bin(l.u64, B::Mul, b, l.k(l.u64, 0x0101_0101_0101_0101)) + let b = l.imm(l.i64, B::BitAnd, r, 0xff); + l.bin(l.i64, B::Mul, b, l.k(l.i64, 0x0101_0101_0101_0101)) } 1 => { - let h = l.imm(l.u64, B::BitAnd, r, 0xffff); - l.bin(l.u64, B::Mul, h, l.k(l.u64, 0x0001_0001_0001_0001)) + let h = l.imm(l.i64, B::BitAnd, r, 0xffff); + l.bin(l.i64, B::Mul, h, l.k(l.i64, 0x0001_0001_0001_0001)) } 2 => { - let w = l.imm(l.u64, B::BitAnd, r, 0xffff_ffff); - l.bin(l.u64, B::BitOr, w, l.imm(l.u64, B::Shl, w, 32)) + let w = l.imm(l.i64, B::BitAnd, r, 0xffff_ffff); + l.bin(l.i64, B::BitOr, w, l.imm(l.i64, B::Shl, w, 32)) } _ => r, }; let at = offset_of!(Cpu, v) + rd as usize * 16; l.store(at, val64); - let hi = if q { val64 } else { l.k(l.u64, 0) }; + let hi = if q { val64 } else { l.k(l.i64, 0) }; l.store(at + 8, hi); } Instruction::SimdExt(a) => { let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); - let vm_lo = l.load(l.u64, at_m); - let vm_hi = l.load(l.u64, at_m + 8); + let vn_lo = l.load(l.i64, at_n); + let vn_hi = l.load(l.i64, at_n + 8); + let vm_lo = l.load(l.i64, at_m); + let vm_hi = l.load(l.i64, at_m + 8); let (lo, hi) = if a.imm == 0 { (vn_lo, vn_hi) } else if a.imm == 8 { @@ -1289,16 +1283,16 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let s = a.imm as u64 * 8; let r = (8 - a.imm as u64) * 8; let lo = l.bin( - l.u64, + l.i64, B::BitOr, - l.imm(l.u64, B::Shr, vn_lo, s), - l.imm(l.u64, B::Shl, vn_hi, r), + l.imm(l.i64, B::Shr, vn_lo, s), + l.imm(l.i64, B::Shl, vn_hi, r), ); let hi = l.bin( - l.u64, + l.i64, B::BitOr, - l.imm(l.u64, B::Shr, vn_hi, s), - l.imm(l.u64, B::Shl, vm_lo, r), + l.imm(l.i64, B::Shr, vn_hi, s), + l.imm(l.i64, B::Shl, vm_lo, r), ); (lo, hi) } else { @@ -1306,16 +1300,16 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let s = shift * 8; let r = (8 - shift) * 8; let lo = l.bin( - l.u64, + l.i64, B::BitOr, - l.imm(l.u64, B::Shr, vn_hi, s), - l.imm(l.u64, B::Shl, vm_lo, r), + l.imm(l.i64, B::Shr, vn_hi, s), + l.imm(l.i64, B::Shl, vm_lo, r), ); let hi = l.bin( - l.u64, + l.i64, B::BitOr, - l.imm(l.u64, B::Shr, vm_lo, s), - l.imm(l.u64, B::Shl, vm_hi, r), + l.imm(l.i64, B::Shr, vm_lo, s), + l.imm(l.i64, B::Shl, vm_hi, r), ); (lo, hi) }; @@ -1325,8 +1319,8 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< Instruction::SimdCompareZero(a) => { let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); + let vn_lo = l.load(l.i64, at_n); + let vn_hi = l.load(l.i64, at_n + 8); let ebits = (1u64 << a.size) * 8; let emask = if ebits == 64 { u64::MAX @@ -1336,18 +1330,18 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let halves = if a.q { 2 } else { 1 }; for half in 0..2 { if half >= halves { - l.store(at_d + half * 8, l.k(l.u64, 0)); + l.store(at_d + half * 8, l.k(l.i64, 0)); continue; } let v = if half == 0 { vn_lo } else { vn_hi }; - let mut acc = l.k(l.u64, 0); + let mut acc = l.k(l.i64, 0); let lanes = 8 >> a.size; for lane in 0..lanes { let shift = lane as u64 * ebits; - let field = l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, v, shift), emask); - let is_zero = l.cmp(C::Eq, field, l.k(l.u64, 0)); - let bits = l.sel(l.u64, is_zero, l.k(l.u64, emask), l.k(l.u64, 0)); - acc = l.bin(l.u64, B::BitOr, acc, l.imm(l.u64, B::Shl, bits, shift)); + let field = l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, v, shift), emask); + let is_zero = l.cmp(C::Eq, field, l.k(l.i64, 0)); + let bits = l.sel(l.i64, is_zero, l.k(l.i64, emask), l.k(l.i64, 0)); + acc = l.bin(l.i64, B::BitOr, acc, l.imm(l.i64, B::Shl, bits, shift)); } l.store(at_d + half * 8, acc); } @@ -1357,24 +1351,24 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< // byte selects a table byte; out-of-range indices leave zero for // `tbl` and keep the destination byte for `tbx`. let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let vm_lo = l.load(l.u64, at_m); - let vm_hi = l.load(l.u64, at_m + 8); + let vm_lo = l.load(l.i64, at_m); + let vm_hi = l.load(l.i64, at_m + 8); let regs = a.len as usize + 1; let mut table = Vec::with_capacity(regs * 2); for i in 0..regs { let reg = (a.rn as usize + i) % 32; let at = offset_of!(Cpu, v) + reg * 16; - table.push(l.load(l.u64, at)); - table.push(l.load(l.u64, at + 8)); + table.push(l.load(l.i64, at)); + table.push(l.load(l.i64, at + 8)); } let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; let old_lo = if a.extend { - Some(l.load(l.u64, at_d)) + Some(l.load(l.i64, at_d)) } else { None }; let old_hi = if a.extend { - Some(l.load(l.u64, at_d + 8)) + Some(l.load(l.i64, at_d + 8)) } else { None }; @@ -1382,37 +1376,37 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let limit = (16 * regs) as u64; for half in 0..2 { if half >= halves { - l.store(at_d + half * 8, l.k(l.u64, 0)); + l.store(at_d + half * 8, l.k(l.i64, 0)); continue; } let indices = if half == 0 { vm_lo } else { vm_hi }; let old = if half == 0 { old_lo } else { old_hi }; - let mut acc = l.k(l.u64, 0); + let mut acc = l.k(l.i64, 0); for byte in 0..8u64 { let shift = byte * 8; let index = - l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, indices, shift), 0xff); - let in_range = l.cmp(C::Lt, index, l.k(l.u64, limit)); - let mut value = l.k(l.u64, 0); - let sel = |c: Value, x: Value, y: Value| l.sel(l.u64, c, x, y); + l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, indices, shift), 0xff); + let in_range = l.cmp(C::Ult, index, l.k(l.i64, limit)); + let mut value = l.k(l.i64, 0); + let sel = |c: Value, x: Value, y: Value| l.sel(l.i64, c, x, y); for (b, entry) in table.iter().enumerate() { let here = l.cmp( C::Eq, - l.bin(l.u64, B::Shr, index, l.k(l.u64, 3)), - l.k(l.u64, b as u64), + l.bin(l.i64, B::Shr, index, l.k(l.i64, 3)), + l.k(l.i64, b as u64), ); let byte_at = l.imm( - l.u64, + l.i64, B::BitAnd, l.bin( - l.u64, + l.i64, B::Shr, *entry, l.bin( - l.u64, + l.i64, B::Shl, - l.imm(l.u64, B::BitAnd, index, 7), - l.k(l.u64, 3), + l.imm(l.i64, B::BitAnd, index, 7), + l.k(l.i64, 3), ), ), 0xff, @@ -1425,14 +1419,14 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< in_range, value, l.imm( - l.u64, + l.i64, B::BitAnd, - l.imm(l.u64, B::Shr, old.unwrap(), shift), + l.imm(l.i64, B::Shr, old.unwrap(), shift), 0xff, ), ); } - acc = l.bin(l.u64, B::BitOr, acc, l.imm(l.u64, B::Shl, value, shift)); + acc = l.bin(l.i64, B::BitOr, acc, l.imm(l.i64, B::Shl, value, shift)); } l.store(at_d + half * 8, acc); } @@ -1440,28 +1434,28 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< Instruction::SimdFmov(a) => { // Pure bit move; no FP arithmetic involved. if a.to_fp { - let v = l.reg(l.u64, a.rn, true); + let v = l.reg(l.i64, a.rn, true); let bits = if a.double { v } else { - l.imm(l.u64, B::BitAnd, v, 0xffff_ffff) + l.imm(l.i64, B::BitAnd, v, 0xffff_ffff) }; let at = offset_of!(Cpu, v) + a.rd as usize * 16; l.store(at, bits); - l.store(at + 8, l.k(l.u64, 0)); + l.store(at + 8, l.k(l.i64, 0)); } else { let at = offset_of!(Cpu, v) + a.rn as usize * 16; - let v = l.load(l.u64, at); + let v = l.load(l.i64, at); let bits = if a.double { v } else { - l.imm(l.u64, B::BitAnd, v, 0xffff_ffff) + l.imm(l.i64, B::BitAnd, v, 0xffff_ffff) }; l.put(a.rd, bits); } } Instruction::SimdAlu(a) => { - use crate::aarch64::decode::SimdAluOp; + use volt_target::aarch64::decode::SimdAluOp; // Bitwise logical operations work on whole halves, lane width free. if matches!( a.op, @@ -1486,36 +1480,36 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< }; let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); + let vn_lo = l.load(l.i64, at_n); + let vn_hi = l.load(l.i64, at_n + 8); let halves = if a.q { 2 } else { 1 }; let mut total = if a.op == SimdAluOp::UMinV { - l.k(l.u64, emask) + l.k(l.i64, emask) } else { - l.k(l.u64, 0) + l.k(l.i64, 0) }; for half in 0..halves { let v = if half == 0 { vn_lo } else { vn_hi }; for lane in 0..(8 >> a.size) { let shift = lane as u64 * ebits; let field = - l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, v, shift), emask); + l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, v, shift), emask); total = match a.op { SimdAluOp::UMinV => { - l.sel(l.u64, l.cmp(C::Lt, field, total), field, total) + l.sel(l.i64, l.cmp(C::Ult, field, total), field, total) } SimdAluOp::UMaxV => { - l.sel(l.u64, l.cmp(C::Gt, field, total), field, total) + l.sel(l.i64, l.cmp(C::Ugt, field, total), field, total) } - _ => l.bin(l.u64, B::Add, total, field), + _ => l.bin(l.i64, B::Add, total, field), }; } } l.store( at_d, - l.bin(l.u64, B::BitAnd, total, l.k(l.u64, result_mask)), + l.bin(l.i64, B::BitAnd, total, l.k(l.i64, result_mask)), ); - l.store(at_d + 8, l.k(l.u64, 0)); + l.store(at_d + 8, l.k(l.i64, 0)); } else if matches!( a.op, SimdAluOp::And @@ -1532,45 +1526,45 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; for half in 0..2 { if half == 1 && !a.q { - l.store(at_d + 8, l.k(l.u64, 0)); + l.store(at_d + 8, l.k(l.i64, 0)); continue; } - let x = l.load(l.u64, at_n + half * 8); - let y = l.load(l.u64, at_m + half * 8); - let not = |v| l.bin(l.u64, B::BitXor, v, l.k(l.u64, u64::MAX)); + let x = l.load(l.i64, at_n + half * 8); + let y = l.load(l.i64, at_m + half * 8); + let not = |v| l.bin(l.i64, B::BitXor, v, l.k(l.i64, u64::MAX)); let r = match a.op { - SimdAluOp::And => l.bin(l.u64, B::BitAnd, x, y), - SimdAluOp::Bic => l.bin(l.u64, B::BitAnd, x, not(y)), - SimdAluOp::Orr => l.bin(l.u64, B::BitOr, x, y), - SimdAluOp::Orn => l.bin(l.u64, B::BitOr, x, not(y)), - SimdAluOp::Eor => l.bin(l.u64, B::BitXor, x, y), + SimdAluOp::And => l.bin(l.i64, B::BitAnd, x, y), + SimdAluOp::Bic => l.bin(l.i64, B::BitAnd, x, not(y)), + SimdAluOp::Orr => l.bin(l.i64, B::BitOr, x, y), + SimdAluOp::Orn => l.bin(l.i64, B::BitOr, x, not(y)), + SimdAluOp::Eor => l.bin(l.i64, B::BitXor, x, y), // The destination is both a source mask and the // merge base for BSL/BIT/BIF. _ => { - let d = l.load(l.u64, at_d + half * 8); + let d = l.load(l.i64, at_d + half * 8); match a.op { SimdAluOp::Bsl => l.bin( - l.u64, + l.i64, B::BitOr, - l.bin(l.u64, B::BitAnd, x, d), - l.bin(l.u64, B::BitAnd, y, not(d)), + l.bin(l.i64, B::BitAnd, x, d), + l.bin(l.i64, B::BitAnd, y, not(d)), ), _ => { let selected = match a.op { SimdAluOp::Bit => l.bin( - l.u64, + l.i64, B::BitAnd, - l.bin(l.u64, B::BitXor, d, x), + l.bin(l.i64, B::BitXor, d, x), y, ), _ => l.bin( - l.u64, + l.i64, B::BitAnd, - l.bin(l.u64, B::BitXor, d, x), + l.bin(l.i64, B::BitXor, d, x), not(y), ), }; - l.bin(l.u64, B::BitXor, d, selected) + l.bin(l.i64, B::BitXor, d, selected) } } } @@ -1581,10 +1575,10 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); - let vm_lo = l.load(l.u64, at_m); - let vm_hi = l.load(l.u64, at_m + 8); + let vn_lo = l.load(l.i64, at_n); + let vn_hi = l.load(l.i64, at_n + 8); + let vm_lo = l.load(l.i64, at_m); + let vm_hi = l.load(l.i64, at_m + 8); let need_d = matches!(a.op, SimdAluOp::Mla | SimdAluOp::Mls); // Pairwise forms consume adjacent lanes of the full source pair: // lane 0/1 of Vn and Vm, then lane 2/3, and so on. @@ -1605,67 +1599,64 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let s64 = l.signed(Width::X64); - let scmp = - |op: C, x: Value, y: Value| l.cmp(op, l.cv(s64, x), l.cv(s64, y)); - let ssel = |c: Value, x: Value, y: Value| l.sel(l.u64, c, x, y); + let ssel = |c: Value, x: Value, y: Value| l.sel(l.i64, c, x, y); let sources = [ - l.load(l.u64, at_n), - l.load(l.u64, at_n + 8), - l.load(l.u64, at_m), - l.load(l.u64, at_m + 8), + l.load(l.i64, at_n), + l.load(l.i64, at_n + 8), + l.load(l.i64, at_m), + l.load(l.i64, at_m + 8), ]; // Each source is a 128-bit vector. Lane `2k` pairs with lane // `2k + 1` and the result lands in lane `k`; the reduction of Vn // fills the destination's low half and Vm its high half. let per_source = (128 / ebits) as usize; let reduced_lanes = per_source / 2; - let mut out = [l.k(l.u64, 0); 2]; + let mut out = [l.k(l.i64, 0); 2]; for (which, half) in [(0usize, 0usize), (2usize, 1usize)] { - let mut acc = l.k(l.u64, 0); + let mut acc = l.k(l.i64, 0); for k in 0..reduced_lanes { let lanes_per_dw = (64 / ebits) as usize; let read = |lane: usize| { let double = lane / lanes_per_dw; let shift = (lane % lanes_per_dw) as u64 * ebits; l.imm( - l.u64, + l.i64, B::BitAnd, - l.imm(l.u64, B::Shr, sources[which + double], shift), + l.imm(l.i64, B::Shr, sources[which + double], shift), emask, ) }; let lo = read(2 * k); let hi = read(2 * k + 1); let r = match a.op { - SimdAluOp::AddP => l.bin(l.u64, B::Add, lo, hi), - SimdAluOp::UMaxP => ssel(l.cmp(C::Gt, lo, hi), lo, hi), - SimdAluOp::UMinP => ssel(l.cmp(C::Lt, lo, hi), lo, hi), - SimdAluOp::SMaxP => ssel(scmp(C::Gt, lo, hi), lo, hi), - _ => ssel(scmp(C::Lt, lo, hi), lo, hi), + SimdAluOp::AddP => l.bin(l.i64, B::Add, lo, hi), + SimdAluOp::UMaxP => ssel(l.cmp(C::Ugt, lo, hi), lo, hi), + SimdAluOp::UMinP => ssel(l.cmp(C::Ult, lo, hi), lo, hi), + SimdAluOp::SMaxP => ssel(l.cmp(C::Sgt, lo, hi), lo, hi), + _ => ssel(l.cmp(C::Slt, lo, hi), lo, hi), }; - let masked = l.bin(l.u64, B::BitAnd, r, l.k(l.u64, emask)); + let masked = l.bin(l.i64, B::BitAnd, r, l.k(l.i64, emask)); // per_source lanes reduce to exactly 64 bits, so the // reduced lanes always pack into one doubleword. acc = l.bin( - l.u64, + l.i64, B::BitOr, acc, - l.imm(l.u64, B::Shl, masked, k as u64 * ebits), + l.imm(l.i64, B::Shl, masked, k as u64 * ebits), ); } out[half] = acc; } l.store(at_d, out[0]); - l.store(at_d + 8, if a.q { out[1] } else { l.k(l.u64, 0) }); + l.store(at_d + 8, if a.q { out[1] } else { l.k(l.i64, 0) }); } else { let vd_lo = if need_d { - Some(l.load(l.u64, at_d)) + Some(l.load(l.i64, at_d)) } else { None }; let vd_hi = if need_d { - Some(l.load(l.u64, at_d + 8)) + Some(l.load(l.i64, at_d + 8)) } else { None }; @@ -1675,126 +1666,119 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } else { (1u64 << ebits) - 1 }; - let s64 = l.signed(Width::X64); let smax = u64::MAX >> (65 - ebits); let smin = !smax; let halves = if a.q { 2 } else { 1 }; for half in 0..2 { if half >= halves { - l.store(at_d + half * 8, l.k(l.u64, 0)); + l.store(at_d + half * 8, l.k(l.i64, 0)); continue; } let vn = if half == 0 { vn_lo } else { vn_hi }; let vm = if half == 0 { vm_lo } else { vm_hi }; let vd = if half == 0 { vd_lo } else { vd_hi }; - let mut acc = l.k(l.u64, 0); + let mut acc = l.k(l.i64, 0); let lanes = 8 >> a.size; for lane in 0..lanes { let shift = lane as u64 * ebits; let fld = |v| { - l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, v, shift), emask) + l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, v, shift), emask) }; let ext = |v: Value| { - let up = l.bin(s64, B::Shl, l.cv(s64, v), l.k(s64, 64 - ebits)); - l.imm(s64, B::Shr, up, 64 - ebits) + let up = l.bin(l.i64, B::Shl, v, l.k(l.i64, 64 - ebits)); + l.imm(l.i64, B::Sar, up, 64 - ebits) }; let an = fld(vn); let bn = fld(vm); let sa = ext(an); let sb = ext(bn); let dn = vd.map(|v| fld(v)); - let scmp = |op: C, x: Value, y: Value| { - l.cmp(op, l.cv(s64, x), l.cv(s64, y)) - }; - let ssel = |c: Value, x: Value, y: Value| l.sel(l.u64, c, x, y); + let ssel = |c: Value, x: Value, y: Value| l.sel(l.i64, c, x, y); let ssat = |v: Value| { - let over = scmp(C::Gt, v, l.k(l.u64, smax)); - let under = scmp(C::Lt, v, l.k(l.u64, smin)); - ssel(over, l.k(l.u64, smax), ssel(under, l.k(l.u64, smin), v)) + let over = l.cmp(C::Sgt, v, l.k(l.i64, smax)); + let under = l.cmp(C::Slt, v, l.k(l.i64, smin)); + ssel(over, l.k(l.i64, smax), ssel(under, l.k(l.i64, smin), v)) }; let r = match a.op { - SimdAluOp::Add => l.bin(l.u64, B::Add, an, bn), - SimdAluOp::Sub => l.bin(l.u64, B::Sub, an, bn), - SimdAluOp::Mul => l.bin(l.u64, B::Mul, an, bn), + SimdAluOp::Add => l.bin(l.i64, B::Add, an, bn), + SimdAluOp::Sub => l.bin(l.i64, B::Sub, an, bn), + SimdAluOp::Mul => l.bin(l.i64, B::Mul, an, bn), SimdAluOp::Mla => l.bin( - l.u64, + l.i64, B::Add, dn.unwrap(), - l.bin(l.u64, B::Mul, an, bn), + l.bin(l.i64, B::Mul, an, bn), ), SimdAluOp::Mls => l.bin( - l.u64, + l.i64, B::Sub, dn.unwrap(), - l.bin(l.u64, B::Mul, an, bn), + l.bin(l.i64, B::Mul, an, bn), ), SimdAluOp::CmEq => { - ssel(l.cmp(C::Eq, an, bn), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmGt => { - ssel(scmp(C::Gt, sa, sb), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmGe => { - ssel(scmp(C::Ge, sa, sb), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmHi => { - ssel(l.cmp(C::Gt, an, bn), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmHs => { - ssel(l.cmp(C::Ge, an, bn), l.k(l.u64, emask), l.k(l.u64, 0)) + ssel(l.cmp(C::Eq, an, bn), l.k(l.i64, emask), l.k(l.i64, 0)) } + SimdAluOp::CmGt => ssel( + l.cmp(C::Sgt, sa, sb), + l.k(l.i64, emask), + l.k(l.i64, 0), + ), + SimdAluOp::CmGe => ssel( + l.cmp(C::Sge, sa, sb), + l.k(l.i64, emask), + l.k(l.i64, 0), + ), + SimdAluOp::CmHi => ssel( + l.cmp(C::Ugt, an, bn), + l.k(l.i64, emask), + l.k(l.i64, 0), + ), + SimdAluOp::CmHs => ssel( + l.cmp(C::Uge, an, bn), + l.k(l.i64, emask), + l.k(l.i64, 0), + ), SimdAluOp::CmTst => ssel( l.cmp( C::Ne, - l.bin(l.u64, B::BitAnd, an, bn), - l.k(l.u64, 0), + l.bin(l.i64, B::BitAnd, an, bn), + l.k(l.i64, 0), ), - l.k(l.u64, emask), - l.k(l.u64, 0), + l.k(l.i64, emask), + l.k(l.i64, 0), ), - SimdAluOp::UMin => ssel(l.cmp(C::Lt, an, bn), an, bn), - SimdAluOp::UMax => ssel(l.cmp(C::Gt, an, bn), an, bn), - SimdAluOp::SMin => ssel(scmp(C::Lt, sa, sb), an, bn), - SimdAluOp::SMax => ssel(scmp(C::Gt, sa, sb), an, bn), + SimdAluOp::UMin => ssel(l.cmp(C::Ult, an, bn), an, bn), + SimdAluOp::UMax => ssel(l.cmp(C::Ugt, an, bn), an, bn), + SimdAluOp::SMin => ssel(l.cmp(C::Slt, sa, sb), an, bn), + SimdAluOp::SMax => ssel(l.cmp(C::Sgt, sa, sb), an, bn), SimdAluOp::UAbd => { - let d = l.bin(l.u64, B::Sub, an, bn); + let d = l.bin(l.i64, B::Sub, an, bn); ssel( - l.cmp(C::Ge, an, bn), + l.cmp(C::Uge, an, bn), d, - l.bin(l.u64, B::Sub, l.k(l.u64, 0), d), + l.bin(l.i64, B::Sub, l.k(l.i64, 0), d), ) } SimdAluOp::SAbd => { - let d = l.bin(s64, B::Sub, sa, sb); - let neg = l.bin(s64, B::Sub, l.k(s64, 0), d); - l.cv( - l.u64, - ssel( - scmp(C::Ge, sa, sb), - l.cv(l.u64, d), - l.cv(l.u64, neg), - ), - ) - } - SimdAluOp::SQAdd => { - ssat(l.cv(l.u64, l.bin(s64, B::Add, sa, sb))) + let d = l.bin(l.i64, B::Sub, sa, sb); + let neg = l.bin(l.i64, B::Sub, l.k(l.i64, 0), d); + ssel(l.cmp(C::Sge, sa, sb), d, neg) } + SimdAluOp::SQAdd => ssat(l.bin(l.i64, B::Add, sa, sb)), SimdAluOp::UQAdd => { - let s = l.bin(l.u64, B::Add, an, bn); - let carry = l.cmp(C::Lt, s, an); - let big = l.cmp(C::Gt, s, l.k(l.u64, emask)); + let s = l.bin(l.i64, B::Add, an, bn); + let carry = l.cmp(C::Ult, s, an); + let big = l.cmp(C::Ugt, s, l.k(l.i64, emask)); ssel( - l.bin(l.u64, B::BitOr, carry, big), - l.k(l.u64, emask), + l.bin(l.i64, B::BitOr, carry, big), + l.k(l.i64, emask), s, ) } - SimdAluOp::SQSub => { - ssat(l.cv(l.u64, l.bin(s64, B::Sub, sa, sb))) - } + SimdAluOp::SQSub => ssat(l.bin(l.i64, B::Sub, sa, sb)), SimdAluOp::UQSub => { - let borrow = l.cmp(C::Gt, bn, an); - ssel(borrow, l.k(l.u64, 0), l.bin(l.u64, B::Sub, an, bn)) + let borrow = l.cmp(C::Ugt, bn, an); + ssel(borrow, l.k(l.i64, 0), l.bin(l.i64, B::Sub, an, bn)) } SimdAluOp::SHAdd | SimdAluOp::UHAdd @@ -1808,19 +1792,16 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< ); // Signed forms operate on sign-extended lanes, which // keeps the halving shift meaningful for negatives. - let (op_a, op_b) = if signed_op { - (l.cv(l.u64, sa), l.cv(l.u64, sb)) - } else { - (an, bn) - }; + let (op_a, op_b) = + if signed_op { (sa, sb) } else { (an, bn) }; let shr = |v: Value| { if signed_op { - l.cv(l.u64, l.imm(s64, B::Shr, l.cv(s64, v), 1)) + l.imm(l.i64, B::Sar, v, 1) } else { - l.imm(l.u64, B::Shr, v, 1) + l.imm(l.i64, B::Shr, v, 1) } }; - let k = l.bin(l.u64, B::Sub, shr(op_a), shr(op_b)); + let k = l.bin(l.i64, B::Sub, shr(op_a), shr(op_b)); let k = if matches!( a.op, SimdAluOp::SHAdd @@ -1828,23 +1809,23 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< | SimdAluOp::SRHAdd | SimdAluOp::URHAdd ) { - l.bin(l.u64, B::Add, shr(op_a), shr(op_b)) + l.bin(l.i64, B::Add, shr(op_a), shr(op_b)) } else { k }; - let a0 = l.bin(l.u64, B::BitAnd, op_a, l.k(l.u64, 1)); - let b0 = l.bin(l.u64, B::BitAnd, op_b, l.k(l.u64, 1)); + let a0 = l.bin(l.i64, B::BitAnd, op_a, l.k(l.i64, 1)); + let b0 = l.bin(l.i64, B::BitAnd, op_b, l.k(l.i64, 1)); let corr = match a.op { SimdAluOp::SHAdd | SimdAluOp::UHAdd => { - l.bin(l.u64, B::BitAnd, a0, b0) + l.bin(l.i64, B::BitAnd, a0, b0) } SimdAluOp::SRHAdd | SimdAluOp::URHAdd => { - l.bin(l.u64, B::BitOr, a0, b0) + l.bin(l.i64, B::BitOr, a0, b0) } _ => l.bin( - l.u64, + l.i64, B::BitAnd, - l.bin(l.u64, B::BitXor, a0, l.k(l.u64, 1)), + l.bin(l.i64, B::BitXor, a0, l.k(l.i64, 1)), b0, ), }; @@ -1855,9 +1836,9 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< | SimdAluOp::SRHAdd | SimdAluOp::URHAdd ) { - l.bin(l.u64, B::Add, k, corr) + l.bin(l.i64, B::Add, k, corr) } else { - l.bin(l.u64, B::Sub, k, corr) + l.bin(l.i64, B::Sub, k, corr) } } SimdAluOp::SSHl @@ -1881,79 +1862,71 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< ); // Shift amount: the lane's low byte, sign-extended to // 64 bits (negative means a right shift). - let amt = l.cv( - s64, - l.bin( - s64, - B::Shl, - l.cv( - s64, - l.bin(l.u64, B::BitAnd, bn, l.k(l.u64, 0xff)), - ), - l.k(s64, 56), - ), + let amt = l.bin( + l.i64, + B::Shl, + l.bin(l.i64, B::BitAnd, bn, l.k(l.i64, 0xff)), + l.k(l.i64, 56), ); - let amt = l.cv(s64, l.bin(s64, B::Shr, amt, l.k(s64, 56))); - let left = scmp(C::Ge, amt, l.k(l.u64, 0)); + let amt = l.bin(l.i64, B::Sar, amt, l.k(l.i64, 56)); + let left = l.cmp(C::Sge, amt, l.k(l.i64, 0)); let n = ssel( left, - l.cv(l.u64, amt), - l.cv(l.u64, l.bin(s64, B::Sub, l.k(s64, 0), amt)), + amt, + l.bin(l.i64, B::Sub, l.k(l.i64, 0), amt), ); - let shl = |v: Value, k: Value| l.bin(l.u64, B::Shl, v, k); - let sar = |v: Value, k: Value| { - l.cv(l.u64, l.bin(s64, B::Shr, l.cv(s64, v), k)) - }; - let shr = |v: Value, k: Value| l.bin(l.u64, B::Shr, v, k); + let shl = |v: Value, k: Value| l.bin(l.i64, B::Shl, v, k); + let sar = |v: Value, k: Value| l.bin(l.i64, B::Sar, v, k); + let shr = |v: Value, k: Value| l.bin(l.i64, B::Shr, v, k); // Beyond the lane width: a plain right shift keeps the // sign fill (or zero), a rounding shift collapses to zero, // and a saturating left shift pins to the lane bound. let sign_fill = if signed_op { ssel( - scmp(C::Lt, sa, l.k(l.u64, 0)), - l.k(l.u64, emask), - l.k(l.u64, 0), + l.cmp(C::Slt, sa, l.k(l.i64, 0)), + l.k(l.i64, emask), + l.k(l.i64, 0), ) } else { - l.k(l.u64, 0) + l.k(l.i64, 0) }; let sat_fill = if signed_op { ssel( - scmp(C::Lt, sa, l.k(l.u64, 0)), - l.k(l.u64, smin), - l.k(l.u64, smax), + l.cmp(C::Slt, sa, l.k(l.i64, 0)), + l.k(l.i64, smin), + l.k(l.i64, smax), ) } else { - l.k(l.u64, emask) + l.k(l.i64, emask) }; let far = ssel( left, - if saturating { sat_fill } else { l.k(l.u64, 0) }, - if rounding { l.k(l.u64, 0) } else { sign_fill }, + if saturating { sat_fill } else { l.k(l.i64, 0) }, + if rounding { l.k(l.i64, 0) } else { sign_fill }, ); // Shift when the amount stays inside the lane width. // Clamping to `ebits - 1` keeps the host shift defined. let kc = ssel( - l.cmp(C::Ge, n, l.k(l.u64, ebits)), - l.k(l.u64, ebits - 1), + l.cmp(C::Uge, n, l.k(l.i64, ebits)), + l.k(l.i64, ebits - 1), n, ); let rnd = if rounding { shl( - l.k(l.u64, 1), - l.bin(l.u64, B::Sub, kc, l.k(l.u64, 1)), + l.k(l.i64, 1), + l.bin(l.i64, B::Sub, kc, l.k(l.i64, 1)), ) } else { - l.k(l.u64, 0) + l.k(l.i64, 0) }; - let opnd = if signed_op { l.cv(l.u64, sa) } else { an }; + let opnd = if signed_op { sa } else { an }; let t = if rounding { // The rounding constant must not leak out of the lane. l.bin( - l.u64, + l.i64, B::BitAnd, - l.bin(l.u64, B::Add, opnd, rnd), - l.k(l.u64, emask), + l.bin(l.i64, B::Add, opnd, rnd), + l.k(l.i64, emask), ) } else { opnd @@ -1968,7 +1941,7 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let shifted = shl(opnd, kc); let moved = ssel(left, shifted, right); let moved = - ssel(l.cmp(C::Ge, n, l.k(l.u64, ebits)), far, moved); + ssel(l.cmp(C::Uge, n, l.k(l.i64, ebits)), far, moved); if !saturating { moved } else if signed_op { @@ -1983,33 +1956,33 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< l.bin( l.boolean, B::BitOr, - scmp(C::Gt, shifted, l.k(l.u64, smax)), - scmp(C::Lt, shifted, l.k(l.u64, smin)), + l.cmp(C::Sgt, shifted, l.k(l.i64, smax)), + l.cmp(C::Slt, shifted, l.k(l.i64, smin)), ) }; let sat = ssel( - scmp(C::Lt, sa, l.k(l.u64, 0)), - l.k(l.u64, smin), - l.k(l.u64, smax), + l.cmp(C::Slt, sa, l.k(l.i64, 0)), + l.k(l.i64, smin), + l.k(l.i64, smax), ); ssel(left, ssel(ov, sat, moved), moved) } else { let ov = if ebits == 64 { l.cmp(C::Ne, shr(shifted, kc), an) } else { - l.cmp(C::Gt, shifted, l.k(l.u64, emask)) + l.cmp(C::Ugt, shifted, l.k(l.i64, emask)) }; - ssel(left, ssel(ov, l.k(l.u64, emask), moved), moved) + ssel(left, ssel(ov, l.k(l.i64, emask), moved), moved) } } - _ => l.k(l.u64, 0), + _ => l.k(l.i64, 0), }; - let masked = l.bin(l.u64, B::BitAnd, r, l.k(l.u64, emask)); + let masked = l.bin(l.i64, B::BitAnd, r, l.k(l.i64, emask)); acc = l.bin( - l.u64, + l.i64, B::BitOr, acc, - l.imm(l.u64, B::Shl, masked, shift), + l.imm(l.i64, B::Shl, masked, shift), ); } l.store(at_d + half * 8, acc); @@ -2034,9 +2007,9 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let (bytes, chunk_count) = if a.bytes == 16 { (16, 4) } else { (8, 2) }; let starts: [Value; 4] = [ address, - l.imm(l.u64, B::Add, address, 8), - l.imm(l.u64, B::Add, address, 16), - l.imm(l.u64, B::Add, address, 24), + l.imm(l.i64, B::Add, address, 8), + l.imm(l.i64, B::Add, address, 16), + l.imm(l.i64, B::Add, address, 24), ]; // stores: (reg_offset_base, half) per chunk let src: [(usize, u64); 4] = if bytes == 16 { @@ -2055,17 +2028,17 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< if store { for i in 0..chunk_count { let host = hosts[i].unwrap(); - l.store_ptr(host, l.load(l.u64, src[i].0)); + l.store_ptr(host, l.load(l.i64, src[i].0)); } } else { for i in 0..chunk_count { let host = hosts[i].unwrap(); - l.store(src[i].0, l.load_ptr(l.u64, host)); + l.store(src[i].0, l.load_ptr(l.i64, host)); } // zero the untransferred half of each 128-bit register if bytes == 8 { - l.store(at_rt + 8, l.k(l.u64, 0)); - l.store(at_rt2 + 8, l.k(l.u64, 0)); + l.store(at_rt + 8, l.k(l.i64, 0)); + l.store(at_rt2 + 8, l.k(l.i64, 0)); } } if let Some((rn, value)) = post { @@ -2100,8 +2073,8 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< _ => Size::Double, }; let chunk_count = if wide { 2 } else { 1 }; - let starts: [Value; 2] = [address, l.imm(l.u64, B::Add, address, 8)]; - let chunk_ty = l.uint((a.bytes.min(8) as u16) * 8); + let starts: [Value; 2] = [address, l.imm(l.i64, B::Add, address, 8)]; + let chunk_ty = l.int((a.bytes.min(8) as u16) * 8); let src: [usize; 2] = [at_rt, at_rt + 8]; let mut hosts: [Option; 2] = [None; 2]; for i in 0..chunk_count { @@ -2114,20 +2087,20 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< if store { for i in 0..chunk_count { let host = hosts[i].unwrap(); - l.store_ptr(host, l.cv(chunk_ty, l.load(l.u64, src[i]))); + l.store_ptr(host, l.cv(chunk_ty, l.load(l.i64, src[i]))); } } else { for i in 0..chunk_count { let host = hosts[i].unwrap(); - l.store(src[i], l.cv(l.u64, l.load_ptr(chunk_ty, host))); + l.store(src[i], l.cv(l.i64, l.load_ptr(chunk_ty, host))); } // Scalar loads zero the whole upper register, not just the // untransferred half. if a.bytes <= 8 { - l.store(at_rt + 8, l.k(l.u64, 0)); + l.store(at_rt + 8, l.k(l.i64, 0)); if a.bytes < 8 { let kept = (1u64 << (a.bytes * 8)) - 1; - let lo = l.imm(l.u64, B::BitAnd, l.load(l.u64, at_rt), kept); + let lo = l.imm(l.i64, B::BitAnd, l.load(l.i64, at_rt), kept); l.store(at_rt, lo); } } @@ -2142,23 +2115,29 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } } Instruction::Nop - | Instruction::CacheOp - | Instruction::Dsb - | Instruction::Dmb + | Instruction::Hint(_) + | Instruction::Prefetch(_) + | Instruction::PrefetchLiteral(_) + | Instruction::CacheOp(_) + | Instruction::Dsb(_) + | Instruction::Dmb(_) | Instruction::Yield => {} Instruction::Wfe => l.trap(pc.wrapping_add(4), Trap::Wfe), Instruction::Brk(imm) => { - l.store(offset_of!(Cpu, address), l.k(l.u64, u64::from(imm))); + l.store(offset_of!(Cpu, address), l.k(l.i64, u64::from(imm))); l.trap(pc, Trap::Brk) } - Instruction::Isb => l.trap(pc.wrapping_add(4), Trap::Isb), - Instruction::Tlbi => l.trap(pc.wrapping_add(4), Trap::Tlbi), + Instruction::Isb(_) => l.trap(pc.wrapping_add(4), Trap::Isb), + Instruction::Tlbi(_) => l.trap(pc.wrapping_add(4), Trap::Tlbi), Instruction::DcZva(rt) => { - l.store(offset_of!(Cpu, address), l.reg(l.u64, rt, true)); + l.store(offset_of!(Cpu, address), l.reg(l.i64, rt, true)); l.trap(pc.wrapping_add(4), Trap::DcZva) } Instruction::Eret => l.trap(pc.wrapping_add(4), Trap::Eret), - Instruction::Svc => l.trap(pc.wrapping_add(4), Trap::Svc), + Instruction::Svc(imm) => { + l.store(offset_of!(Cpu, address), l.k(l.i64, u64::from(imm))); + l.trap(pc.wrapping_add(4), Trap::Svc) + } Instruction::Psci => l.trap(pc.wrapping_add(4), Trap::Psci), Instruction::Wfi => l.trap(pc.wrapping_add(4), Trap::Wfi), } @@ -2168,7 +2147,7 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } } if !ended { - l.ret(l.k(l.u64, pc)) + l.ret(l.k(l.i64, pc)) } Ok(Block::new(native::compile_owned(l.f.into_inner())?)) } @@ -2178,7 +2157,7 @@ fn setup_pair(l: &Lower, address: Value, a: Pair) { setup_memory(l, address, a.size, a.rt, a.signed, a.op); l.store( offset_of!(Cpu, second_address), - l.imm(l.u64, B::Add, address, a.size as u64), + l.imm(l.i64, B::Add, address, a.size as u64), ); l.byte(offset_of!(Cpu, second_width), a.size as u64); l.byte(offset_of!(Cpu, second_dest), a.rt2 as u64); @@ -2186,7 +2165,7 @@ fn setup_pair(l: &Lower, address: Value, a: Pair) { l.byte(offset_of!(Cpu, second_signed), a.signed as u64) } if a.op == MemoryOp::Store { - l.store(offset_of!(Cpu, second_value), l.reg(l.u64, a.rt2, true)) + l.store(offset_of!(Cpu, second_value), l.reg(l.i64, a.rt2, true)) } l.byte(offset_of!(Cpu, second_pending), 1); } @@ -2198,7 +2177,7 @@ fn setup_memory(l: &Lower, address: Value, size: Size, rt: u8, signed: SignExten l.byte(offset_of!(Cpu, load_signed), signed as u64) } if op == MemoryOp::Store { - l.store(offset_of!(Cpu, value), l.reg(l.u64, rt, true)) + l.store(offset_of!(Cpu, value), l.reg(l.i64, rt, true)) } } @@ -2213,10 +2192,7 @@ fn lower_variable(l: &Lower, a: Variable) { match a.op { VariableOp::Lsl => l.bin(t, B::Shl, lhs, distance), VariableOp::Lsr => l.bin(t, B::Shr, lhs, distance), - VariableOp::Asr => { - let s = l.signed(a.width); - l.cv(t, l.bin(s, B::Shr, l.cv(s, lhs), l.cv(s, distance))) - } + VariableOp::Asr => l.bin(t, B::Sar, lhs, distance), VariableOp::Ror => { let back = l.imm( t, @@ -2240,12 +2216,11 @@ fn lower_variable(l: &Lower, a: Variable) { let by_zero = l.cmp(C::Eq, rhs, zero); if a.op == VariableOp::Udiv { let safe = l.sel(t, by_zero, one, rhs); - l.sel(t, by_zero, zero, l.bin(t, B::Div, lhs, safe)) + l.sel(t, by_zero, zero, l.bin(t, B::UDiv, lhs, safe)) } else { let minus_one = l.cmp(C::Eq, rhs, l.k(t, u64::MAX)); let safe = l.sel(t, minus_one, one, l.sel(t, by_zero, one, rhs)); - let s = l.signed(a.width); - let quotient = l.cv(t, l.bin(s, B::Div, l.cv(s, lhs), l.cv(s, safe))); + let quotient = l.bin(t, B::SDiv, lhs, safe); let negated = l.bin(t, B::Sub, zero, lhs); l.sel(t, by_zero, zero, l.sel(t, minus_one, negated, quotient)) } @@ -2384,7 +2359,7 @@ fn lower_unary(l: &Lower, a: Unary) { } } fn lower_high(l: &Lower, a: MultiplyHigh) { - let t = l.u64; + let t = l.i64; let x = l.reg(t, a.rn, true); let y = l.reg(t, a.rm, true); let xl = l.imm(t, B::BitAnd, x, 0xffffffff); @@ -2556,14 +2531,14 @@ fn lower_system(l: &Lower, a: System) { _ => None, }; if let Some(level) = level { - let current = l.cv(l.u64, l.load(l.u8, at + offset_of!(CpuSystem, el))); - let at_level = l.cmp(C::Eq, current, l.k(l.u64, level)); + let current = l.cv(l.i64, l.load(l.i8, at + offset_of!(CpuSystem, el))); + let at_level = l.cmp(C::Eq, current, l.k(l.i64, level)); let live = if level <= 1 { - let el1 = l.cmp(C::Eq, current, l.k(l.u64, 1)); + let el1 = l.cmp(C::Eq, current, l.k(l.i64, 1)); let selected = l.cmp( if level == 0 { C::Eq } else { C::Ne }, - l.load(l.u8, at + offset_of!(CpuSystem, spsel)), - l.k(l.u8, 0), + l.load(l.i8, at + offset_of!(CpuSystem, spsel)), + l.k(l.i8, 0), ); let el1_selected = l.bin(l.boolean, B::BitAnd, el1, selected); if level == 0 { @@ -2581,32 +2556,32 @@ fn lower_system(l: &Lower, a: System) { l.addr(at + offset_of!(CpuSystem, sp_el) + level as usize * 8), ); if a.read { - l.put(a.rt, l.load_ptr(l.u64, p)) + l.put(a.rt, l.load_ptr(l.i64, p)) } else { - l.store_ptr(p, l.reg(l.u64, a.rt, true)) + l.store_ptr(p, l.reg(l.i64, a.rt, true)) } return; } if let Some((el1, el2)) = vhe_banks(a.register) { // HCR_EL2.E2H: EL1 register names accessed from EL2 reach the EL2 banks. - let current = l.cv(l.u64, l.load(l.u8, at + offset_of!(CpuSystem, el))); - let el2_mode = l.cmp(C::Eq, current, l.k(l.u64, 2)); + let current = l.cv(l.i64, l.load(l.i8, at + offset_of!(CpuSystem, el))); + let el2_mode = l.cmp(C::Eq, current, l.k(l.i64, 2)); let e2h = l.cmp( C::Ne, l.imm( - l.u64, + l.i64, B::BitAnd, - l.load(l.u64, at + offset_of!(CpuSystem, hcr_el2)), + l.load(l.i64, at + offset_of!(CpuSystem, hcr_el2)), super::cpu::HCR_E2H, ), - l.k(l.u64, 0), + l.k(l.i64, 0), ); let redirect = l.bin(l.boolean, B::BitAnd, el2_mode, e2h); let p = l.sel(l.ptr, redirect, l.addr(at + el2), l.addr(at + el1)); if a.read { - l.put(a.rt, l.load_ptr(l.u64, p)); + l.put(a.rt, l.load_ptr(l.i64, p)); } else { - l.store_ptr(p, l.reg(l.u64, a.rt, true)); + l.store_ptr(p, l.reg(l.i64, a.rt, true)); } return; } @@ -2614,9 +2589,9 @@ fn lower_system(l: &Lower, a: System) { CurrentEl => l.put( a.rt, l.imm( - l.u64, + l.i64, B::Shl, - l.cv(l.u64, l.load(l.u8, at + offset_of!(CpuSystem, el))), + l.cv(l.i64, l.load(l.i8, at + offset_of!(CpuSystem, el))), 2, ), ), @@ -2624,29 +2599,29 @@ fn lower_system(l: &Lower, a: System) { // PSTATE.PAN lives in bit 22 of the register form. let pan = at + offset_of!(CpuSystem, pan); if a.read { - l.put(a.rt, l.imm(l.u64, B::Shl, l.load(l.u64, pan), 22)); + l.put(a.rt, l.imm(l.i64, B::Shl, l.load(l.i64, pan), 22)); } else if a.register == PanSet { - l.store(pan, l.k(l.u64, u64::from(a.immediate & 1))); + l.store(pan, l.k(l.i64, u64::from(a.immediate & 1))); } else { - let bit = l.imm(l.u64, B::Shr, l.reg(l.u64, a.rt, true), 22); - l.store(pan, l.imm(l.u64, B::BitAnd, bit, 1)); + let bit = l.imm(l.i64, B::Shr, l.reg(l.i64, a.rt, true), 22); + l.store(pan, l.imm(l.i64, B::BitAnd, bit, 1)); } } RazWi | AppleZero => { if a.read { - l.put(a.rt, l.k(l.u64, 0)); + l.put(a.rt, l.k(l.i64, 0)); } } SpSel | SpSelSet => { - let selected = l.load(l.u8, at + offset_of!(CpuSystem, spsel)); + let selected = l.load(l.i8, at + offset_of!(CpuSystem, spsel)); if a.read { - l.put(a.rt, l.cv(l.u64, selected)); + l.put(a.rt, l.cv(l.i64, selected)); return; } let new = if a.register == SpSelSet { - l.k(l.u8, u64::from(a.immediate & 1)) + l.k(l.i8, u64::from(a.immediate & 1)) } else { - l.cv(l.u8, l.imm(l.u64, B::BitAnd, l.reg(l.u64, a.rt, true), 1)) + l.cv(l.i8, l.imm(l.i64, B::BitAnd, l.reg(l.i64, a.rt, true), 1)) }; // The live stack pointer belongs to bank `SP_EL1` when SPSel is 1 // and `SP_EL0` when it is 0. Park it in its bank, then load the @@ -2656,22 +2631,22 @@ fn lower_system(l: &Lower, a: System) { let bank = |select: Value| { l.sel( l.ptr, - l.cmp(C::Ne, select, l.k(l.u8, 0)), + l.cmp(C::Ne, select, l.k(l.i8, 0)), l.addr(banks + 8), l.addr(banks), ) }; - let live = l.load(l.u64, Lower::reg_at(31)); + let live = l.load(l.i64, Lower::reg_at(31)); l.store_ptr(bank(selected), live); - let incoming = l.load_ptr(l.u64, bank(new)); + let incoming = l.load_ptr(l.i64, bank(new)); l.store(Lower::reg_at(31), incoming); l.store(at + offset_of!(CpuSystem, spsel), new); } DaifSet | DaifClear => { let p = at + offset_of!(CpuSystem, daif); - let before = l.load(l.u64, p); + let before = l.load(l.i64, p); let updated = l.imm( - l.u64, + l.i64, if a.register == DaifSet { B::BitOr } else { @@ -2688,14 +2663,14 @@ fn lower_system(l: &Lower, a: System) { } Nzcv => { if a.read { - l.put(a.rt, l.cv(l.u64, l.load(l.u32, offset_of!(Cpu, flags)))) + l.put(a.rt, l.cv(l.i64, l.load(l.i32, offset_of!(Cpu, flags)))) } else { l.store( offset_of!(Cpu, flags), l.imm( - l.u32, + l.i32, B::BitAnd, - l.cv(l.u32, l.reg(l.u64, a.rt, true)), + l.cv(l.i32, l.reg(l.i64, a.rt, true)), 0xf0000000, ), ) @@ -2703,35 +2678,35 @@ fn lower_system(l: &Lower, a: System) { } OslsrEl1 => { let either = l.bin( - l.u64, + l.i64, B::BitOr, - l.load(l.u64, at + offset_of!(CpuSystem, oslar_el1)), - l.load(l.u64, at + offset_of!(CpuSystem, osdlr_el1)), + l.load(l.i64, at + offset_of!(CpuSystem, oslar_el1)), + l.load(l.i64, at + offset_of!(CpuSystem, osdlr_el1)), ); - let locked = l.cmp(C::Ne, either, l.k(l.u64, 0)); - l.put(a.rt, l.sel(l.u64, locked, l.k(l.u64, 2), l.k(l.u64, 0))); + let locked = l.cmp(C::Ne, either, l.k(l.i64, 0)); + l.put(a.rt, l.sel(l.i64, locked, l.k(l.i64, 2), l.k(l.i64, 0))); } CcsidrEl1 => { // CSSELR bit 0 is InD, bits 3:1 the level - 1; L2 has no separate I-side. use super::identification::{CCSIDR_L1D, CCSIDR_L1I, CCSIDR_L2}; let select = l.imm( - l.u64, + l.i64, B::BitAnd, - l.load(l.u64, at + offset_of!(CpuSystem, csselr_el1)), + l.load(l.i64, at + offset_of!(CpuSystem, csselr_el1)), 0xf, ); - let is = |n: u64| l.cmp(C::Eq, select, l.k(l.u64, n)); - let l1_instruction = l.sel(l.u64, is(1), l.k(l.u64, CCSIDR_L1I), l.k(l.u64, CCSIDR_L2)); + let is = |n: u64| l.cmp(C::Eq, select, l.k(l.i64, n)); + let l1_instruction = l.sel(l.i64, is(1), l.k(l.i64, CCSIDR_L1I), l.k(l.i64, CCSIDR_L2)); l.put( a.rt, - l.sel(l.u64, is(0), l.k(l.u64, CCSIDR_L1D), l1_instruction), + l.sel(l.i64, is(0), l.k(l.i64, CCSIDR_L1D), l1_instruction), ); } CntvTvalEl0 | CntpTvalEl0 => { // The timer value is the signed 32-bit distance to the compare value: // reads give the low 32 bits of `CVAL - count`, writes `CVAL = count + sext(value)`. // The physical timer shares the virtual count, as CNTVOFF_EL2 is zero without EL2. - let counter = l.load(l.u64, at + offset_of!(CpuSystem, cntvct_el0)); + let counter = l.load(l.i64, at + offset_of!(CpuSystem, cntvct_el0)); let compare = at + if a.register == CntpTvalEl0 { offset_of!(CpuSystem, cntp_cval_el0) @@ -2739,37 +2714,37 @@ fn lower_system(l: &Lower, a: System) { offset_of!(CpuSystem, cntv_cval_el0) }; if a.read { - let distance = l.bin(l.u64, B::Sub, l.load(l.u64, compare), counter); - l.put(a.rt, l.imm(l.u64, B::BitAnd, distance, 0xffff_ffff)); + let distance = l.bin(l.i64, B::Sub, l.load(l.i64, compare), counter); + l.put(a.rt, l.imm(l.i64, B::BitAnd, distance, 0xffff_ffff)); } else { - let delta = l.extend(l.reg(l.u64, a.rt, true), Extend::Sxtw); - l.store(compare, l.bin(l.u64, B::Add, counter, delta)); + let delta = l.extend(l.reg(l.i64, a.rt, true), Extend::Sxtw); + l.store(compare, l.bin(l.i64, B::Add, counter, delta)); } } CntpctEl0 => { // Writes are ignored: the counter is not writable from software. if a.read { - l.put(a.rt, l.load(l.u64, at + offset_of!(CpuSystem, cntvct_el0))); + l.put(a.rt, l.load(l.i64, at + offset_of!(CpuSystem, cntvct_el0))); } } IccIar1El1 => { // No group 1 interrupt is delivered by this model, so acknowledge reads spurious. if a.read { - l.put(a.rt, l.k(l.u64, 1023)); + l.put(a.rt, l.k(l.i64, 1023)); } } IccEoir1El1 => {} Daif => { let p = at + offset_of!(CpuSystem, daif); if a.read { - l.put(a.rt, l.imm(l.u64, B::Shl, l.load(l.u64, p), 6)) + l.put(a.rt, l.imm(l.i64, B::Shl, l.load(l.i64, p), 6)) } else { l.store( p, l.imm( - l.u64, + l.i64, B::BitAnd, - l.imm(l.u64, B::Shr, l.reg(l.u64, a.rt, true), 6), + l.imm(l.i64, B::Shr, l.reg(l.i64, a.rt, true), 6), 0xf, ), ); @@ -2777,32 +2752,32 @@ fn lower_system(l: &Lower, a: System) { } _ => { if let Some(constant) = super::identification::value(a.register) { - l.put(a.rt, l.k(l.u64, constant)); + l.put(a.rt, l.k(l.i64, constant)); return; } let offset = system_offset(a.register).expect("decoded system register has storage"); let p = at + offset; if a.read { let v = if a.register == CntvCtlEl0 { - let counter = l.load(l.u64, at + offset_of!(CpuSystem, cntvct_el0)); - let compare = l.load(l.u64, at + offset_of!(CpuSystem, cntv_cval_el0)); - let reached = l.cmp(C::Ge, counter, compare); + let counter = l.load(l.i64, at + offset_of!(CpuSystem, cntvct_el0)); + let compare = l.load(l.i64, at + offset_of!(CpuSystem, cntv_cval_el0)); + let reached = l.cmp(C::Uge, counter, compare); l.bin( - l.u64, + l.i64, B::BitOr, - l.load(l.u64, p), - l.sel(l.u64, reached, l.k(l.u64, 4), l.k(l.u64, 0)), + l.load(l.i64, p), + l.sel(l.i64, reached, l.k(l.i64, 4), l.k(l.i64, 0)), ) } else { - l.load(l.u64, p) + l.load(l.i64, p) }; l.put(a.rt, v); } else { - let from = l.reg(l.u64, a.rt, true); + let from = l.reg(l.i64, a.rt, true); l.store( p, if a.register == CntvCtlEl0 { - l.imm(l.u64, B::BitAnd, from, 3) + l.imm(l.i64, B::BitAnd, from, 3) } else { from }, @@ -2847,6 +2822,169 @@ mod tests { assert_eq!(cpu.x[3], 1 << 63); } #[test] + fn signed_and_unsigned_integer_ops_select_distinct_semantics() { + // sdiv/udiv/asrv (w and x), smulh/umulh, sxtw, sbfx, asr #imm, then + // `cmp` + `cset` for each signed/unsigned condition on w and x. + let words = [ + 0x1ac10c02, 0x1ac10803, 0x1ac12804, 0x9ac12805, 0x9b417c06, 0x9bc17c07, 0x93407c08, + 0x93442c09, 0x13037c0a, 0x6b01001f, 0x1a9fa7eb, 0x1a9fb7ec, 0x1a9f27ed, 0x1a9f97ee, + 0x1a9fd7ef, 0x1a9fc7f0, 0x1a9f87f1, 0xeb01001f, 0x9a9fa7f2, 0x9a9f27f3, 0x9a9fd7f4, + ]; + let block = compile(0, &bytes(&words)).unwrap(); + let pairs: [(u64, u64); 9] = [ + (0xffff_ffff_8000_0005, 3), + (0x8000_0000, 0xffff_ffff), + (0x7fff_ffff_ffff_ffff, 0x8000_0000_0000_0000), + (0x8000_0000_0000_0000, u64::MAX), + (3, 0), + (7, 7), + (0xffff_ffff_0000_0007, 0x0000_0001_0000_0009), + (0x8000_0001, 2), + (2, 0x8000_0001), + ]; + for (a, b) in pairs { + let mut cpu = Cpu::default(); + cpu.x[0] = a; + cpu.x[1] = b; + block.run(&mut cpu); + let (w, wb) = (a as u32, b as u32); + let (sw, swb) = (w as i32, wb as i32); + let sdiv = if wb == 0 { + 0 + } else { + sw.wrapping_div(swb) as u32 + }; + let udiv = if wb == 0 { 0 } else { w / wb }; + assert_eq!(cpu.x[2], sdiv as u64, "sdiv w {a:#x} {b:#x}"); + assert_eq!(cpu.x[3], udiv as u64, "udiv w {a:#x} {b:#x}"); + assert_eq!(cpu.x[4], (sw >> (wb & 31)) as u32 as u64, "asr w"); + assert_eq!(cpu.x[5], ((a as i64) >> (b & 63)) as u64, "asr x"); + assert_eq!( + cpu.x[6], + ((a as i64 as i128 * b as i64 as i128) >> 64) as u64, + "smulh" + ); + assert_eq!(cpu.x[7], ((a as u128 * b as u128) >> 64) as u64, "umulh"); + assert_eq!(cpu.x[8], w as i32 as i64 as u64, "sxtw"); + assert_eq!(cpu.x[9], ((a >> 4) as u8) as i8 as i64 as u64, "sbfx"); + assert_eq!(cpu.x[10], (sw >> 3) as u32 as u64, "asr w imm"); + // Registers 11-17 come from the 32-bit compare, 18-20 from the 64-bit one. + let w_flags = [ + sw < swb, + sw >= swb, + w < wb, + w > wb, + sw > swb, + sw <= swb, + w <= wb, + ]; + for (i, want) in w_flags.into_iter().enumerate() { + assert_eq!(cpu.x[11 + i], want as u64, "w cond {i} {a:#x} {b:#x}"); + } + let (sa, sb) = (a as i64, b as i64); + assert_eq!(cpu.x[18], (sa < sb) as u64, "x lt"); + assert_eq!(cpu.x[19], (a < b) as u64, "x lo"); + assert_eq!(cpu.x[20], (sa > sb) as u64, "x gt"); + } + } + #[test] + fn signed_and_unsigned_vector_lanes_select_distinct_semantics() { + // smax.16b, sabd.8h, shadd.4s, sshl.4s, sqadd.16b, cmgt.2d, cmhi.2d. + let words = [ + 0x4e216403, 0x4e617404, 0x4ea10405, 0x4ea14406, 0x4e210c07, 0x4ee13409, 0x6ee1340a, + ]; + let block = compile(0, &bytes(&words)).unwrap(); + let a: [u8; 16] = [ + 0x80, 0x7f, 0x01, 0xff, 0x10, 0x90, 0x00, 0x7e, 0x85, 0x05, 0xfe, 0x40, 0x7f, 0x80, + 0xc0, 0x20, + ]; + let b: [u8; 16] = [ + 0x01, 0x80, 0xff, 0x01, 0x90, 0x10, 0x00, 0x7f, 0x05, 0x85, 0x02, 0xc0, 0x7f, 0x7f, + 0x40, 0xe0, + ]; + let pack = |x: [u8; 16]| { + let v = u128::from_le_bytes(x); + [v as u64, (v >> 64) as u64] + }; + let mut cpu = Cpu::default(); + cpu.v[0] = pack(a); + cpu.v[1] = pack(b); + block.run(&mut cpu); + let out = + |r: usize| (u128::from(cpu.v[r][1]) << 64 | u128::from(cpu.v[r][0])).to_le_bytes(); + let lanes16 = |x: [u8; 16]| -> Vec { + x.chunks(2) + .map(|c| i16::from_le_bytes([c[0], c[1]])) + .collect() + }; + let lanes32 = |x: [u8; 16]| -> Vec { + x.chunks(4) + .map(|c| i32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + }; + // smax.16b + let want: Vec = (0..16) + .map(|i| (a[i] as i8).max(b[i] as i8) as u8) + .collect(); + assert_eq!(out(3).to_vec(), want, "smax"); + // sabd.8h + let want: Vec = lanes16(a) + .iter() + .zip(lanes16(b)) + .flat_map(|(x, y)| ((*x as i32 - y as i32).unsigned_abs() as u16).to_le_bytes()) + .collect(); + assert_eq!(out(4).to_vec(), want, "sabd"); + // shadd.4s + let want: Vec = lanes32(a) + .iter() + .zip(lanes32(b)) + .flat_map(|(x, y)| (((*x as i64 + y as i64) >> 1) as i32).to_le_bytes()) + .collect(); + assert_eq!(out(5).to_vec(), want, "shadd"); + // sshl.4s: the shift amount is the signed low byte of each lane of b. + let want: Vec = lanes32(a) + .iter() + .zip(lanes32(b)) + .flat_map(|(x, y)| { + let n = y as i8 as i32; + let r = if n >= 32 { + 0 + } else if n >= 0 { + x << n + } else if n > -32 { + x >> -n + } else if *x < 0 { + -1 + } else { + 0 + }; + r.to_le_bytes() + }) + .collect(); + assert_eq!(out(6).to_vec(), want, "sshl"); + // sqadd.16b + let want: Vec = (0..16) + .map(|i| (a[i] as i8).saturating_add(b[i] as i8) as u8) + .collect(); + assert_eq!(out(7).to_vec(), want, "sqadd"); + // cmgt.2d / cmhi.2d + for (r, signed) in [(9, true), (10, false)] { + for lane in 0..2 { + let (x, y) = (cpu.v[0][lane], cpu.v[1][lane]); + let gt = if signed { + (x as i64) > (y as i64) + } else { + x > y + }; + assert_eq!( + cpu.v[r][lane], + if gt { u64::MAX } else { 0 }, + "cmp {r} {lane}" + ); + } + } + } + #[test] fn pair_post_index_records_deferred_access_and_writeback() { // LDP X0,X1,[X2],#16. let block = compile(0x8000, &bytes(&[0xa8c10440])).unwrap(); diff --git a/src/emulator/src/aarch64/cpu.rs b/src/emulator/src/aarch64/cpu.rs index 9fd37f1..6a2d32b 100644 --- a/src/emulator/src/aarch64/cpu.rs +++ b/src/emulator/src/aarch64/cpu.rs @@ -1,5 +1,5 @@ -use super::decode::{AtomicKind, SignExtend}; -use super::simd_struct::StructDesc; +use volt_target::aarch64::decode::{AtomicKind, SignExtend}; +use volt_target::aarch64::simd_struct::StructDesc; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] @@ -159,13 +159,13 @@ pub struct Cpu { pub apple_bank: ImpDefBank, } -/// `S3__c15` registers, indexed by `decode::apple_bank_slot`. +/// `S3__c15` registers, indexed by `volt_target::aarch64::decode::apple_bank_slot`. #[derive(Clone, Debug)] -pub struct ImpDefBank(pub [u64; crate::aarch64::decode::APPLE_BANK_SLOTS]); +pub struct ImpDefBank(pub [u64; volt_target::aarch64::decode::APPLE_BANK_SLOTS]); impl Default for ImpDefBank { fn default() -> Self { - Self([0; crate::aarch64::decode::APPLE_BANK_SLOTS]) + Self([0; volt_target::aarch64::decode::APPLE_BANK_SLOTS]) } } @@ -235,8 +235,8 @@ pub struct System { pub hcr_el2: u64, pub tpidr_el2: u64, pub cntvoff_el2: u64, - /// Apple IMP-DEF configuration registers, indexed by `decode::SystemRegister::Apple`. - pub apple: [u64; crate::aarch64::decode::APPLE_SLOTS], + /// Apple IMP-DEF configuration registers, indexed by `SystemRegister::Apple`. + pub apple: [u64; volt_target::aarch64::decode::APPLE_SLOTS], /// `MIDR_EL1`, `ID_AA64PFR0_EL1` and `ID_AA64MMFR0_EL1`: the identity this CPU /// advertises, set by the board. pub midr_el1: u64, @@ -306,7 +306,7 @@ impl Default for System { cntp_ctl_el0: 0, cntp_cval_el0: 0, actlr_el1: 0, - apple: [0; crate::aarch64::decode::APPLE_SLOTS], + apple: [0; volt_target::aarch64::decode::APPLE_SLOTS], midr_el1: crate::aarch64::identification::DEFAULT_MIDR_EL1, id_aa64pfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64PFR0_EL1, id_aa64mmfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64MMFR0_EL1, diff --git a/src/emulator/src/aarch64/fp.rs b/src/emulator/src/aarch64/fp.rs index 176b5b6..d3224a0 100644 --- a/src/emulator/src/aarch64/fp.rs +++ b/src/emulator/src/aarch64/fp.rs @@ -11,6 +11,7 @@ //! rounding is always to nearest, ties to even), and the `FPSR` exception flags. use super::Cpu; +use volt_target::aarch64::decode::expand_immediate; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum Precision { @@ -146,19 +147,6 @@ fn precision(ty: u32) -> Option { } } -/// The 32- or 64-bit pattern of the 8-bit floating-point immediate `imm8` -/// (`VFPExpandImm`), as `fmov` and the vector `fmov` take it. -pub fn expand_immediate(imm8: u64, double: bool) -> u64 { - let sign = imm8 >> 7; - let b = (imm8 >> 6) & 1; - let fraction = imm8 & 0x3f; - if double { - (sign << 63) | ((b ^ 1) << 62) | (b * 0xff << 54) | (fraction << 48) - } else { - (sign << 31) | ((b ^ 1) << 30) | (b * 0x1f << 25) | (fraction << 19) - } -} - fn parse(word: u32) -> Option { let rd = (word & 31) as u8; let rn = ((word >> 5) & 31) as u8; diff --git a/src/emulator/src/aarch64/host.rs b/src/emulator/src/aarch64/host.rs index 291d488..9cc6da1 100644 --- a/src/emulator/src/aarch64/host.rs +++ b/src/emulator/src/aarch64/host.rs @@ -3,6 +3,7 @@ //! Floating point and the wider integer SIMD families live in [`fp`] and [`simd`]. use super::{Cpu, fp, gxf, pauth, simd}; +use volt_target::aarch64::decode::{DecodeError, Instruction, decode}; /// Whether [`run`] implements `word`. pub fn is_supported(word: u32) -> bool { @@ -12,6 +13,41 @@ pub fn is_supported(word: u32) -> bool { || gxf::is_supported(word) } +/// What the emulator does with one instruction word: the host runs the words [`run`] +/// implements, and the JIT compiles every other instruction the decoder recognises. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Dispatch { + /// A structured instruction for the JIT. + Guest(Instruction), + /// A word for [`run`]. It ends a block, the same as a trap does. + Host(u32), +} + +impl Dispatch { + /// Whether this word ends a translated block; see `Instruction::terminates_with`. + pub fn terminates_with(self, inline_memory: bool) -> bool { + match self { + Self::Guest(instruction) => instruction.terminates_with(inline_memory), + Self::Host(_) => true, + } + } + + /// [`Self::terminates_with`] without inline memory. + pub fn terminates(self) -> bool { + self.terminates_with(false) + } +} + +/// Classify `word` for execution. A word [`is_supported`] names is never compiled, +/// even where the decoder has a structured form for it. +pub fn dispatch(word: u32) -> Result { + if is_supported(word) { + Ok(Dispatch::Host(word)) + } else { + decode(word).map(Dispatch::Guest) + } +} + /// Run `word` on `cpu`. `word` must satisfy [`is_supported`]. pub fn run(cpu: &mut Cpu, word: u32) { if fp::is_supported(word) { diff --git a/src/emulator/src/aarch64/identification.rs b/src/emulator/src/aarch64/identification.rs index dec538f..87fd5c5 100644 --- a/src/emulator/src/aarch64/identification.rs +++ b/src/emulator/src/aarch64/identification.rs @@ -1,4 +1,4 @@ -use super::decode::SystemRegister; +use volt_target::aarch64::decode::SystemRegister; #[allow(non_upper_case_globals)] pub const counter_hz: u64 = 62_500_000; /// Cache hierarchy `CLIDR_EL1` reports: split L1, unified L2, 64-byte lines. diff --git a/src/emulator/src/aarch64/mod.rs b/src/emulator/src/aarch64/mod.rs index f284009..407a3b3 100644 --- a/src/emulator/src/aarch64/mod.rs +++ b/src/emulator/src/aarch64/mod.rs @@ -2,7 +2,6 @@ pub mod ahead; pub mod cache; pub mod compile; pub mod cpu; -pub mod decode; pub mod exception; pub mod fp; pub mod gxf; @@ -15,4 +14,3 @@ pub mod translate; pub use cache::Cache; pub use cpu::Cpu; -pub use decode::Instruction; diff --git a/src/emulator/src/aarch64/simd_struct.rs b/src/emulator/src/aarch64/simd_struct.rs index 9d1c9ab..81f7dc0 100644 --- a/src/emulator/src/aarch64/simd_struct.rs +++ b/src/emulator/src/aarch64/simd_struct.rs @@ -6,51 +6,7 @@ //! [`execute`] against its own memory, one element at a time, so each backend keeps //! its own translation, device and fault handling. -/// Which elements of the registers a structure access touches. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -#[repr(u8)] -pub enum Shape { - /// Whole vectors: `ld1 {v0.4s - v3.4s}` and the interleaved `ld2`..`ld4`. - #[default] - Multiple, - /// One element of each register: `ld1 {v0.s}[1]`. Other lanes are kept. - Lane, - /// One element broadcast to every lane: `ld1r`. Loads only. - Replicate, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -#[repr(C)] -pub struct StructDesc { - pub store: bool, - pub shape: Shape, - /// Registers touched, starting at `rt` and wrapping from v31 to v0. - pub registers: u8, - /// `ld2`..`ld4`: consecutive memory elements alternate between the registers. - /// Otherwise (`ld1`) each register is a contiguous block. - pub interleaved: bool, - /// Element size in bytes: 1, 2, 4 or 8. - pub esize: u8, - /// Element index of a [`Shape::Lane`] access. - pub lane: u8, - /// A 128-bit vector (otherwise 64-bit, with the upper half zeroed on loads). - pub q: bool, - pub rt: u8, -} - -impl StructDesc { - fn vector_bytes(&self) -> u8 { - if self.q { 16 } else { 8 } - } - /// Bytes of memory the access spans; the post-index immediate advances by this. - pub fn bytes(&self) -> u64 { - let registers = u64::from(self.registers); - match self.shape { - Shape::Multiple => registers * u64::from(self.vector_bytes()), - Shape::Lane | Shape::Replicate => registers * u64::from(self.esize), - } - } -} +use volt_target::aarch64::simd_struct::{Shape, StructDesc}; fn mask(esize: u8) -> u64 { if esize >= 8 { diff --git a/src/emulator/src/system/m1.rs b/src/emulator/src/system/m1.rs index 09cb468..af0af59 100644 --- a/src/emulator/src/system/m1.rs +++ b/src/emulator/src/system/m1.rs @@ -60,7 +60,7 @@ pub fn identify(cpu: &mut crate::aarch64::cpu::Cpu) { // The SPTM monitor polls `S3_6_c15_c12_4` until bit 0 is set before it continues // (`mrs x0, s3_6_c15_c12_4; and x1, x0, #1; cbz x1, `). Nothing here names the // register, so bit 0 reads as set, which is the only value that lets the loop exit. - cpu.apple_bank.0[crate::aarch64::decode::apple_bank_slot(6, 12, 4)] = 1; + cpu.apple_bank.0[volt_target::aarch64::decode::apple_bank_slot(6, 12, 4)] = 1; } /// The SoC devices on the bus: the AIC, the power-state blocks, the console, and the diff --git a/src/emulator/src/system/machine.rs b/src/emulator/src/system/machine.rs index 9c24799..336d4ec 100644 --- a/src/emulator/src/system/machine.rs +++ b/src/emulator/src/system/machine.rs @@ -2,12 +2,12 @@ use super::afdt; use super::fdt; use crate::aarch64::cpu::{Exclusive, Trap}; -use crate::aarch64::decode::SignExtend; use crate::aarch64::simd_struct; use crate::aarch64::{Cache, Cpu}; use crate::aarch64::{exception, translate}; use crate::devices::{bus::Device, gicv2::Gicv2, pl011::Pl011}; use crate::memory::{GuestMemory, MemoryError}; +use volt_target::aarch64::decode::SignExtend; #[derive(Debug)] pub enum Error { @@ -487,7 +487,7 @@ impl Machine { continue; } if let crate::aarch64::compile::Error::Decode { word, pc } = &error - && crate::aarch64::decode::is_undefined(*word) + && volt_target::aarch64::decode::is_undefined(*word) { // An Undefined Instruction exception (EC 0, 32-bit instruction). self.cpu.pc = *pc; @@ -562,12 +562,15 @@ impl Machine { Trap::Svc => { self.cpu.trap = Trap::None; let pc = self.cpu.pc; + // The SVC immediate goes in ISS, as a BRK's does. + let immediate = self.cpu.address as u32; exception::take( &mut self.cpu, exception::Reason { ec: 0x15, status: 0, instruction: true, + iss: immediate, ..Default::default() }, pc, diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index ef2e027..e1ec209 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -14,7 +14,7 @@ pub mod psci; pub mod smp; pub mod xnu; -use crate::aarch64::{Cpu, decode, translate}; +use crate::aarch64::{Cpu, host, translate}; use crate::devices::bus::Device; use crate::devices::pci::PciHost; use crate::devices::virtio_pci::VirtioBlockPci; @@ -767,7 +767,7 @@ fn instruction_diagnostic( let mut bytes = [0; 4]; machine.memory.read(physical, &mut bytes).ok()?; let word = u32::from_le_bytes(bytes); - let decoded = decode::decode(word); + let decoded = host::dispatch(word); last = Some(InstructionDiagnostic { virtual_address: address, physical_address: physical, @@ -882,6 +882,38 @@ mod tests { assert!(report.compiled_blocks >= 3); } + /// A single-precision `fnmadd` is a fused operation, not an `fmov`. Its `Rm` has + /// bits 18:17 set, which the old FMOV pattern accepted because it left bit 24 unchecked. + #[test] + fn fused_fp_operations_are_not_misread_as_fmov() { + let code = [ + 0x1e2e_1000u32, // fmov s0, #1.0 + 0x1e20_1006, // fmov s6, #2.0 + 0x1f26_0002, // fnmadd s2, s0, s6, s0 => -(1) - (1*2) = -3.0 = 0xc0400000 + 0x1e26_0040, // fmov w0, s2 + 0x5318_7c00, // lsr w0, w0, #24 + 0xd2a1_2001, // movz x1, #0x900, lsl #16 (PL011) + 0x3900_0020, // strb w0, [x1] + 0xd2b0_8000, // movz x0, #0x8400, lsl #16 + 0xf280_0100, // movk x0, #8 (PSCI_SYSTEM_OFF) + 0xd400_0002, // hvc #0 + ]; + let mut image = vec![0; 64]; + image[..4].copy_from_slice(&0x14000010u32.to_le_bytes()); + image[4..8].copy_from_slice(&0xd503201fu32.to_le_bytes()); + image[24..32].copy_from_slice(&2u64.to_le_bytes()); + image[56..60].copy_from_slice(&boot::MAGIC.to_le_bytes()); + for w in code { + image.extend_from_slice(&w.to_le_bytes()); + } + let length = image.len() as u64; + image[16..24].copy_from_slice(&length.to_le_bytes()); + let report = + boot_with_serial(&image, Duration::from_secs(10), DEFAULT_CMDLINE, |_| {}).unwrap(); + assert_eq!(report.reason, StopReason::Shutdown, "{report}"); + assert_eq!(report.console, b"\xc0", "high byte of fnmadd s2 result"); + } + fn create_test_image(instructions: &[u32]) -> Vec { let mut image = vec![0; 64]; image[..4].copy_from_slice(&0x14000010u32.to_le_bytes()); diff --git a/src/emulator/src/user/mod.rs b/src/emulator/src/user/mod.rs index 7dae319..35d2e87 100644 --- a/src/emulator/src/user/mod.rs +++ b/src/emulator/src/user/mod.rs @@ -10,11 +10,12 @@ pub mod syscalls; pub use memory::Space; use crate::aarch64::cpu::{Exclusive, Trap}; -use crate::aarch64::decode::{Instruction, SignExtend, SystemRegister, decode}; +use crate::aarch64::host::{Dispatch, dispatch}; use crate::aarch64::{Cache, Cpu}; use crate::memory::{GuestMemory, MemoryError}; use std::ffi::OsString; use std::path::Path; +use volt_target::aarch64::decode::{Instruction, SignExtend, SystemRegister}; #[derive(Debug)] pub enum Error { @@ -150,20 +151,19 @@ fn fetch( } } let word = u32::from_le_bytes(bytes); - let instruction = - decode(word).map_err(|_| crate::aarch64::compile::Error::Decode { word, pc: address })?; - if !user_instruction(&instruction) { - return Err(crate::aarch64::compile::Error::InvalidUserInstruction); - } - if matches!(instruction, Instruction::CacheOp) { - // The decoder groups EL0 and privileged cache operations together. - // Only IC IVAU and DC CVAC/CVAU/CIVAC are permitted in userspace. - let op1 = (word >> 16) & 7; - let crm = (word >> 8) & 15; - let op2 = (word >> 5) & 7; - if op1 != 3 || op2 != 1 || !matches!(crm, 5 | 10 | 11 | 14) { + let dispatched = + dispatch(word).map_err(|_| crate::aarch64::compile::Error::Decode { word, pc: address })?; + if let Dispatch::Guest(instruction) = &dispatched { + if !user_instruction(instruction) { return Err(crate::aarch64::compile::Error::InvalidUserInstruction); } + if let Instruction::CacheOp(sys) = instruction { + // The decoder groups EL0 and privileged cache operations together. + // Only IC IVAU and DC CVAC/CVAU/CIVAC are permitted in userspace. + if sys.op1 != 3 || sys.op2 != 1 || !matches!(sys.crm, 5 | 10 | 11 | 14) { + return Err(crate::aarch64::compile::Error::InvalidUserInstruction); + } + } } if into.len() < 4 { return Err(crate::aarch64::compile::Error::InvalidInstructionLength); @@ -187,7 +187,7 @@ fn user_instruction(instruction: &Instruction) -> bool { }, Instruction::Eret | Instruction::Psci - | Instruction::Tlbi + | Instruction::Tlbi(_) | Instruction::Wfi | Instruction::AddressTranslate(_) => false, _ => true, diff --git a/src/ir/Cargo.toml b/src/ir/Cargo.toml index fe6509b..94168d7 100644 --- a/src/ir/Cargo.toml +++ b/src/ir/Cargo.toml @@ -9,6 +9,3 @@ path = "src/lib.rs" [lints] workspace = true - -[dependencies] -pliron = "0.19.0" diff --git a/src/ir/src/attribute.rs b/src/ir/src/attribute.rs index 34d8412..d0143cb 100644 --- a/src/ir/src/attribute.rs +++ b/src/ir/src/attribute.rs @@ -29,6 +29,5 @@ pub enum Attribute { Noreturn, Cold, Align(u32), - Endian(Endianness), Custom(Custom), } diff --git a/src/ir/src/bitcode.rs b/src/ir/src/bitcode.rs index 7a656ca..a296aa7 100644 --- a/src/ir/src/bitcode.rs +++ b/src/ir/src/bitcode.rs @@ -17,10 +17,11 @@ use alloc::vec::Vec; use super::attribute::{AttrValue, Attribute, Custom, Endianness}; use super::function::{ Alloca, Arith, ArithImm, AtomicOp, AtomicOrdering, AtomicRmw, AtomicScope, AttrTarget, Barrier, - BarrierScope, BinOp, Block, Call, CallIndirect, CmpOp, Compare, Convert, Dot, Extract, - Function, GlobalAddr, InputSigns, Inst, Jump, Load, LowFloatConvert, MatMul, MatMulQuant, - MatMulQuantOut, MatMulScale, MatMulType, NvFp4Convert, Opcode, Prefetch, Reduce, Ret, RetPiece, - Select, Splat, Store, StructNew, Terminator, Unary, UnaryOp, VaArg, VaEnd, VaStart, Value, + BarrierScope, BinOp, Block, Call, CallIndirect, CmpOp, Compare, Convert, ConvertKind, Dot, + Extract, Function, GlobalAddr, InputSigns, Inst, Intrinsic, Jump, Load, LowFloatConvert, + MatMul, MatMulQuant, MatMulQuantOut, MatMulScale, MatMulType, MemFlags, NvFp4Convert, Opcode, + Prefetch, Reduce, Ret, RetPiece, Select, Splat, Store, StructNew, Terminator, Unary, UnaryOp, + VaArg, VaEnd, VaStart, Value, }; use super::low_float::Format as LowFloatFormat; use super::nvfp4::ScaleApplication; @@ -42,7 +43,68 @@ impl core::fmt::Display for MalformedSegment { /// Result alias with the error type as a defaulted parameter. pub type Result = core::result::Result; -const MAGIC: &[u8; 4] = b"VBC1"; +/// `VBC3`: integer types carry no signedness (the type record is the tag and the bit width) +/// and signedness lives on the operation, so `BinOp`, `CmpOp`, `AtomicOp`, `Convert` (a kind +/// byte) and `Dot` (a flag byte) have the tag values pinned below. +const MAGIC: &[u8; 4] = b"VBC3"; + +// The wire tag of each operator enum is its discriminant. Pin them at compile time: the +// decoder is a hardcoded switch, so a future reorder must fail to build instead of silently +// turning one operator into another. +const _: () = { + assert!(BinOp::Add as u8 == 0); + assert!(BinOp::Sub as u8 == 1); + assert!(BinOp::Mul as u8 == 2); + assert!(BinOp::Div as u8 == 3); + assert!(BinOp::Rem as u8 == 4); + assert!(BinOp::BitAnd as u8 == 5); + assert!(BinOp::BitOr as u8 == 6); + assert!(BinOp::BitXor as u8 == 7); + assert!(BinOp::Shl as u8 == 8); + assert!(BinOp::Shr as u8 == 9); + assert!(BinOp::Sar as u8 == 10); + assert!(BinOp::UDiv as u8 == 11); + assert!(BinOp::SDiv as u8 == 12); + assert!(BinOp::URem as u8 == 13); + assert!(BinOp::SRem as u8 == 14); + assert!(BinOp::UMulh as u8 == 15); + assert!(BinOp::SMulh as u8 == 16); + assert!(CmpOp::Eq as u8 == 0); + assert!(CmpOp::Ne as u8 == 1); + assert!(CmpOp::Lt as u8 == 2); + assert!(CmpOp::Le as u8 == 3); + assert!(CmpOp::Gt as u8 == 4); + assert!(CmpOp::Ge as u8 == 5); + assert!(CmpOp::Slt as u8 == 6); + assert!(CmpOp::Sle as u8 == 7); + assert!(CmpOp::Sgt as u8 == 8); + assert!(CmpOp::Sge as u8 == 9); + assert!(CmpOp::Ult as u8 == 10); + assert!(CmpOp::Ule as u8 == 11); + assert!(CmpOp::Ugt as u8 == 12); + assert!(CmpOp::Uge as u8 == 13); + assert!(ConvertKind::Trunc as u8 == 0); + assert!(ConvertKind::Zext as u8 == 1); + assert!(ConvertKind::Sext as u8 == 2); + assert!(ConvertKind::SiToFp as u8 == 3); + assert!(ConvertKind::UiToFp as u8 == 4); + assert!(ConvertKind::FpToSi as u8 == 5); + assert!(ConvertKind::FpToUi as u8 == 6); + assert!(ConvertKind::FpResize as u8 == 7); + assert!(AtomicOp::Add as u8 == 0); + assert!(AtomicOp::SMin as u8 == 1); + assert!(AtomicOp::SMax as u8 == 2); + assert!(AtomicOp::BitAnd as u8 == 3); + assert!(AtomicOp::BitOr as u8 == 4); + assert!(AtomicOp::BitXor as u8 == 5); + assert!(AtomicOp::Exchange as u8 == 6); + assert!(AtomicOp::CompareExchange as u8 == 7); + assert!(AtomicOp::UMin as u8 == 8); + assert!(AtomicOp::UMax as u8 == 9); +}; + +/// Flag bits of a `dot` record's flag byte. +const DOT_FLAG_SIGNED: u8 = 1 << 0; /// Bytes the stream header occupies before the type table: the magic, the /// whole-function flag byte, and the fixed-parameter count. @@ -70,7 +132,7 @@ const ATTR_INLINE: u8 = 0; const ATTR_NORETURN: u8 = 1; const ATTR_COLD: u8 = 2; const ATTR_ALIGN: u8 = 3; -const ATTR_ENDIAN: u8 = 4; +// Attribute tag 4 is retired (it was `endian`, now `MemFlags::endian`); 5 stays pinned. const ATTR_CUSTOM: u8 = 5; /// Namespaced attribute payload tags (stable on the wire). @@ -117,6 +179,7 @@ const OP_DEQUANTIZE_NVFP4: u8 = 28; const OP_QUANTIZE_NVFP4: u8 = 29; const OP_REDUCE: u8 = 30; const OP_SPLAT: u8 = 31; +const OP_INTRINSIC: u8 = 32; /// An `atomic_rmw` record flag bit: the compare operand follows the two /// ordinary operand slots. Written from the field and read back into it, so a @@ -124,6 +187,74 @@ const OP_SPLAT: u8 = 31; /// op/compare agreement `verify` enforces. const ATOMIC_FLAG_COMPARE: u8 = 1 << 0; +/// An `intrinsic` record flag bit: the intrinsic has side effects. +const INTRINSIC_FLAG_SIDE_EFFECTS: u8 = 1 << 0; + +// `MemFlags` wire layout, two bytes (tag values pinned; a decoder rejects any other): +// +// byte 0, flags: +// bit 0 volatile +// bits 1..=2 endian: 0 = Native, 1 = Little, 2 = Big (3 invalid) +// bits 3..=5 ordering: 0 = none, otherwise 1 + AtomicOrdering tag +// (1 Relaxed, 2 Acquire, 3 Release, 4 AcqRel, 5 SeqCst; 6, 7 invalid) +// bits 6..=7 reserved, must be zero +// byte 1, align: 0 = no promise, otherwise log2(align) + 1 (so 1..=32) +const MEM_FLAG_VOLATILE: u8 = 1 << 0; +const MEM_ENDIAN_SHIFT: u8 = 1; +const MEM_ORDER_SHIFT: u8 = 3; + +/// Write a `MemFlags`. A non-zero `align` that is not a power of two has no encoding (the +/// verifier rejects it too), so such a function cannot be serialized. +fn write_mem_flags(w: &mut Writer, mem: &MemFlags) -> Result<()> { + let endian: u8 = match mem.endian { + Endianness::Native => 0, + Endianness::Little => 1, + Endianness::Big => 2, + }; + debug_assert_eq!(AtomicOrdering::Relaxed as u8, 0); + debug_assert_eq!(AtomicOrdering::Acquire as u8, 1); + debug_assert_eq!(AtomicOrdering::Release as u8, 2); + debug_assert_eq!(AtomicOrdering::AcqRel as u8, 3); + debug_assert_eq!(AtomicOrdering::SeqCst as u8, 4); + let ordering: u8 = mem.ordering.map_or(0, |o| 1 + o as u8); + let align: u8 = match mem.align { + 0 => 0, + a if a.is_power_of_two() => a.trailing_zeros() as u8 + 1, + _ => return Err(MalformedSegment), + }; + w.u8((mem.volatile as u8 * MEM_FLAG_VOLATILE) + | endian << MEM_ENDIAN_SHIFT + | ordering << MEM_ORDER_SHIFT); + w.u8(align); + Ok(()) +} + +/// Read a `MemFlags`; the bytes come off an UNTRUSTED stream, so every unknown selector is a +/// recoverable fault and never an invalid enum. +fn read_mem_flags(r: &mut Reader<'_>) -> Result { + let flags = r.u8()?; + let align = r.u8()?; + if flags >> 6 != 0 || align > 32 { + return Err(MalformedSegment); + } + let endian = match (flags >> MEM_ENDIAN_SHIFT) & 0b11 { + 0 => Endianness::Native, + 1 => Endianness::Little, + 2 => Endianness::Big, + _ => return Err(MalformedSegment), + }; + let ordering = match (flags >> MEM_ORDER_SHIFT) & 0b111 { + 0 => None, + n => Some(try_atomic_ordering(n - 1)?), + }; + Ok(MemFlags { + volatile: flags & MEM_FLAG_VOLATILE != 0, + align: if align == 0 { 0 } else { 1u32 << (align - 1) }, + ordering, + endian, + }) +} + struct Writer { bytes: Vec, } @@ -229,7 +360,7 @@ pub fn encode(func: &Function) -> Result> { let insts = func.block_insts(block); w.u32(insts.len() as u32); for inst in insts { - write_inst(&mut w, func, *inst, &serial); + write_inst(&mut w, func, *inst, &serial)?; } write_term(&mut w, func, block, &serial); @@ -308,16 +439,6 @@ fn write_attr(w: &mut Writer, attr: &Attribute) { w.u8(ATTR_ALIGN); w.u32(*a); } - Attribute::Endian(e) => { - w.u8(ATTR_ENDIAN); - // The decoder maps this byte back with a range check, so pin the - // tag values here. This follows the `float` and `ptr` arms of - // `write_type`. - debug_assert_eq!(Endianness::Little as u8, 0); - debug_assert_eq!(Endianness::Big as u8, 1); - debug_assert_eq!(Endianness::Native as u8, 2); - w.u8(*e as u8); - } Attribute::Custom(c) => { w.u8(ATTR_CUSTOM); w.string(&c.namespace); @@ -343,7 +464,6 @@ fn write_type(w: &mut Writer, kind: &TypeKind) { TypeKind::Bool => w.u8(0), TypeKind::Int(i) => { w.u8(1); - w.u8(if i.signed { 0 } else { 1 }); w.u16(i.bits); } TypeKind::Float(f) => { @@ -392,7 +512,7 @@ fn write_type(w: &mut Writer, kind: &TypeKind) { } } -fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { +fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) -> Result<()> { let result = func.inst_result(inst); w.u8(if result.is_some() { 1 } else { 0 }); if let Some(r) = result { @@ -452,6 +572,7 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { } Opcode::Convert(c) => { w.u8(OP_CONVERT); + w.u8(c.kind as u8); w.u32(serial[c.value.0 as usize]); } Opcode::DecodeLowFloat(c) => { @@ -526,19 +647,36 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { &c.ret_pieces, ); } - // `volatile` marks an access the optimizer must not remove, move or - // merge. A stream that drops it turns an MMIO register access into an - // ordinary one. + // The `MemFlags` (volatile, byte order, atomic ordering, alignment) change what the + // access means: a stream that drops `volatile` turns an MMIO register access into an + // ordinary one, and one that drops `ordering` or `endian` silently changes the data + // or the memory model. See `write_mem_flags` for the pinned layout. Opcode::Load(l) => { w.u8(OP_LOAD); w.u32(serial[l.ptr.0 as usize]); - w.u8(l.volatile as u8); + write_mem_flags(w, &l.mem)?; } Opcode::Store(st) => { w.u8(OP_STORE); w.u32(serial[st.value.0 as usize]); w.u32(serial[st.ptr.0 as usize]); - w.u8(st.volatile as u8); + write_mem_flags(w, &st.mem)?; + } + // Operands are the arguments, then the immediate and the effect flag. + Opcode::Intrinsic(i) => { + w.u8(OP_INTRINSIC); + w.u32(i.symbol); + let args = func.value_list(i.args); + w.u32(args.len() as u32); + for a in args { + w.u32(serial[a.0 as usize]); + } + w.u64(i.imm as u64); + w.u8(if i.side_effects { + INTRINSIC_FLAG_SIDE_EFFECTS + } else { + 0 + }); } Opcode::Prefetch(pf) => { w.u8(OP_PREFETCH); @@ -572,14 +710,6 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { w.u8(OP_ATOMIC_RMW); // The decoder maps these three bytes back with range checks, so pin // the tag values here. - debug_assert_eq!(AtomicOp::Add as u8, 0); - debug_assert_eq!(AtomicOp::Min as u8, 1); - debug_assert_eq!(AtomicOp::Max as u8, 2); - debug_assert_eq!(AtomicOp::BitAnd as u8, 3); - debug_assert_eq!(AtomicOp::BitOr as u8, 4); - debug_assert_eq!(AtomicOp::BitXor as u8, 5); - debug_assert_eq!(AtomicOp::Exchange as u8, 6); - debug_assert_eq!(AtomicOp::CompareExchange as u8, 7); debug_assert_eq!(AtomicOrdering::Relaxed as u8, 0); debug_assert_eq!(AtomicOrdering::Acquire as u8, 1); debug_assert_eq!(AtomicOrdering::Release as u8, 2); @@ -604,6 +734,7 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { } Opcode::Dot(d) => { w.u8(OP_DOT); + w.u8(if d.signed { DOT_FLAG_SIGNED } else { 0 }); w.u32(serial[d.acc.0 as usize]); w.u32(serial[d.a.0 as usize]); w.u32(serial[d.b.0 as usize]); @@ -649,6 +780,7 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { w.u8(ga.via_got as u8); } } + Ok(()) } /// Write the quant epilogue, mirroring the Zig field order: relu, out, @@ -967,17 +1099,6 @@ fn read_attr(r: &mut Reader<'_>) -> Result { ATTR_NORETURN => Attribute::Noreturn, ATTR_COLD => Attribute::Cold, ATTR_ALIGN => Attribute::Align(r.u32()?), - ATTR_ENDIAN => { - // The byte comes off an UNTRUSTED stream, so an unknown value is a - // recoverable fault and never an invalid enum. - let order = match r.u8()? { - 0 => Endianness::Little, - 1 => Endianness::Big, - 2 => Endianness::Native, - _ => return Err(MalformedSegment), - }; - Attribute::Endian(order) - } ATTR_CUSTOM => { let namespace = read_string(r)?; let key = read_string(r)?; @@ -1013,9 +1134,8 @@ fn read_type( Ok(match r.u8()? { 0 => func.types.intern(TypeKind::Bool), 1 => { - let signed = r.u8()? == 0; // 0 = signed, 1 = unsigned let bits = r.u16()?; - func.types.intern(TypeKind::Int(IntDesc { signed, bits })) + func.types.intern(TypeKind::Int(IntDesc { bits })) } 2 => { let kind = match r.u8()? { @@ -1241,6 +1361,12 @@ impl Fixup { } Opcode::Reduce(red) => red.vector = next(), Opcode::Splat(sp) => sp.scalar = next(), + Opcode::Intrinsic(i) => { + let args = i.args; + for a in func.value_list_mut(args) { + *a = next(); + } + } Opcode::Matmul(mm) => { mm.a = next(); mm.b = next(); @@ -1391,6 +1517,7 @@ fn requires_result(tag: u8) -> bool { !matches!( tag, OP_STORE + | OP_INTRINSIC | OP_PREFETCH | OP_VA_START | OP_VA_END @@ -1404,7 +1531,7 @@ fn requires_result(tag: u8) -> bool { } /// The result canonicality a record's own has-result type must satisfy: a -/// low-float decode yields f32, an encode yields the unsigned payload width; +/// low-float decode yields f32, an encode yields the payload-width integer; /// anything else is a stream this build cannot read. fn canonical_low_float_result( func: &Function, @@ -1414,16 +1541,16 @@ fn canonical_low_float_result( ) -> bool { match func.types.type_kind(result_type) { TypeKind::Float(f) => decode_direction && *f == FloatKind::F32, - TypeKind::Int(i) => !decode_direction && !i.signed && i.bits == format.payload_bits(), + TypeKind::Int(i) => !decode_direction && i.bits == format.payload_bits(), _ => false, } } -/// Same rule for the NVFP4 records: dequantize yields f32, quantize yields u8. +/// Same rule for the NVFP4 records: dequantize yields f32, quantize yields i8. fn canonical_nvfp4_result(func: &Function, result_type: Type, dequantize_direction: bool) -> bool { match func.types.type_kind(result_type) { TypeKind::Float(f) => dequantize_direction && *f == FloatKind::F32, - TypeKind::Int(i) => !dequantize_direction && !i.signed && i.bits == 8, + TypeKind::Int(i) => !dequantize_direction && i.bits == 8, _ => false, } } @@ -1581,13 +1708,14 @@ fn read_inst( ) } OP_CONVERT => { + let kind = try_convert_kind(r.u8()?)?; fixups.push_slot(r.u32()?); append_res( func, block, serial, rty, - Opcode::Convert(Convert { value: dummy }), + Opcode::Convert(Convert { value: dummy, kind }), ) } OP_DECODE_LOW_FLOAT | OP_ENCODE_LOW_FLOAT => { @@ -1717,32 +1845,61 @@ fn read_inst( } OP_LOAD => { fixups.push_slot(r.u32()?); - let is_volatile = r.u8()? != 0; + let mem = read_mem_flags(r)?; append_res( func, block, serial, rty, - Opcode::Load(Load { - ptr: dummy, - volatile: is_volatile, - }), + Opcode::Load(Load { ptr: dummy, mem }), ) } OP_STORE => { for _ in 0..2 { fixups.push_slot(r.u32()?); } - let is_volatile = r.u8()? != 0; + let mem = read_mem_flags(r)?; func.append_stmt_raw( block, Opcode::Store(Store { value: dummy, ptr: dummy, - volatile: is_volatile, + mem, }), ) } + OP_INTRINSIC => { + let symbol = r.u32()?; + // The stream is UNTRUSTED: the name must exist in the symbol table. + if symbol as usize >= func.symbol_count() { + return Err(MalformedSegment); + } + let n = r.u32()? as usize; + if n > r.remaining() { + return Err(MalformedSegment); + } + for _ in 0..n { + fixups.push_slot(r.u32()?); + } + let imm = r.u64()? as i64; + let flags = r.u8()?; + if flags & !INTRINSIC_FLAG_SIDE_EFFECTS != 0 { + return Err(MalformedSegment); + } + let dummies = alloc::vec![dummy; n]; + let op = Opcode::Intrinsic(Intrinsic { + symbol, + args: func.intern_values(&dummies), + imm, + side_effects: flags & INTRINSIC_FLAG_SIDE_EFFECTS != 0, + }); + // The result is OPTIONAL, exactly as for a `call`. + if has_result { + append_res(func, block, serial, rty, op) + } else { + func.append_stmt_raw(block, op) + } + } OP_PREFETCH => { fixups.push_slot(r.u32()?); func.append_stmt_raw(block, Opcode::Prefetch(Prefetch { ptr: dummy })) @@ -1822,6 +1979,10 @@ fn read_inst( } } OP_DOT => { + let flags = r.u8()?; + if flags & !DOT_FLAG_SIGNED != 0 { + return Err(MalformedSegment); + } for _ in 0..3 { fixups.push_slot(r.u32()?); } @@ -1834,6 +1995,7 @@ fn read_inst( acc: dummy, a: dummy, b: dummy, + signed: flags & DOT_FLAG_SIGNED != 0, }), ) } @@ -1982,13 +2144,15 @@ fn read_inst( fn try_atomic_op(raw: u8) -> Result { Ok(match raw { 0 => AtomicOp::Add, - 1 => AtomicOp::Min, - 2 => AtomicOp::Max, + 1 => AtomicOp::SMin, + 2 => AtomicOp::SMax, 3 => AtomicOp::BitAnd, 4 => AtomicOp::BitOr, 5 => AtomicOp::BitXor, 6 => AtomicOp::Exchange, 7 => AtomicOp::CompareExchange, + 8 => AtomicOp::UMin, + 9 => AtomicOp::UMax, _ => return Err(MalformedSegment), }) } @@ -2026,7 +2190,13 @@ fn try_bin_op(raw: u8) -> Result { 7 => BinOp::BitXor, 8 => BinOp::Shl, 9 => BinOp::Shr, - 10 => BinOp::Mulh, + 10 => BinOp::Sar, + 11 => BinOp::UDiv, + 12 => BinOp::SDiv, + 13 => BinOp::URem, + 14 => BinOp::SRem, + 15 => BinOp::UMulh, + 16 => BinOp::SMulh, _ => return Err(MalformedSegment), }) } @@ -2039,6 +2209,28 @@ fn try_cmp_op(raw: u8) -> Result { 3 => CmpOp::Le, 4 => CmpOp::Gt, 5 => CmpOp::Ge, + 6 => CmpOp::Slt, + 7 => CmpOp::Sle, + 8 => CmpOp::Sgt, + 9 => CmpOp::Sge, + 10 => CmpOp::Ult, + 11 => CmpOp::Ule, + 12 => CmpOp::Ugt, + 13 => CmpOp::Uge, + _ => return Err(MalformedSegment), + }) +} + +fn try_convert_kind(raw: u8) -> Result { + Ok(match raw { + 0 => ConvertKind::Trunc, + 1 => ConvertKind::Zext, + 2 => ConvertKind::Sext, + 3 => ConvertKind::SiToFp, + 4 => ConvertKind::UiToFp, + 5 => ConvertKind::FpToSi, + 6 => ConvertKind::FpToUi, + 7 => ConvertKind::FpResize, _ => return Err(MalformedSegment), }) } @@ -2107,7 +2299,7 @@ mod tests { /// The stream header a hand-built test module starts with: the magic, no /// whole-function flags, and a zero fixed-parameter count. - const TEST_HEADER: &[u8] = b"VBC1\x00\x00\x00\x00\x00"; + const TEST_HEADER: &[u8] = b"VBC3\x00\x00\x00\x00\x00"; /// A structural printer for the oracle: renders a function's blocks, /// params, instructions (operands numbered by canonical serial) and @@ -2173,13 +2365,40 @@ mod tests { vn(c.lhs), vn(c.rhs) )), - Opcode::Convert(c) => out.push_str(&format!("convert {}", vn(c.value))), + Opcode::Convert(c) => { + out.push_str(&format!("convert {} {}", c.kind.name(), vn(c.value))) + } + Opcode::Dot(d) => out.push_str(&format!( + "dot {} {} {} {}", + if d.signed { "signed" } else { "unsigned" }, + vn(d.acc), + vn(d.a), + vn(d.b) + )), Opcode::Call(c) => { out.push_str(&format!("call \"{}\"", func.symbol_name(c.symbol))); for a in func.value_list(c.args) { out.push_str(&format!(" {}", vn(*a))); } } + Opcode::Load(l) => out.push_str(&format!("load {} {:?}", vn(l.ptr), l.mem)), + Opcode::Store(st) => out.push_str(&format!( + "store {} {} {:?}", + vn(st.value), + vn(st.ptr), + st.mem + )), + Opcode::Intrinsic(i) => { + out.push_str(&format!( + "intrinsic \"{}\" imm {} effects {}", + func.symbol_name(i.symbol), + i.imm, + i.side_effects + )); + for a in func.value_list(i.args) { + out.push_str(&format!(" {}", vn(*a))); + } + } Opcode::If(cf) => out.push_str(&format!( "if {} -> block{}, else -> block{}", vn(cf.cond), @@ -2242,10 +2461,7 @@ mod tests { // f(x, y): if x a, _ => unreachable!(), }; - assert_eq!(AtomicOp::Min, reading.op); + assert_eq!(AtomicOp::SMin, reading.op); assert_eq!(AtomicOrdering::AcqRel, reading.ordering); assert_eq!(AtomicScope::Device, reading.scope); assert_eq!(None, reading.compare); @@ -2480,12 +2693,290 @@ mod tests { } #[test] - fn round_trips_a_compare_exchange_and_its_compare_operand() { + fn the_operator_tags_are_pinned_on_the_wire() { + let bins = [ + (0, BinOp::Add), + (1, BinOp::Sub), + (2, BinOp::Mul), + (3, BinOp::Div), + (4, BinOp::Rem), + (5, BinOp::BitAnd), + (6, BinOp::BitOr), + (7, BinOp::BitXor), + (8, BinOp::Shl), + (9, BinOp::Shr), + (10, BinOp::Sar), + (11, BinOp::UDiv), + (12, BinOp::SDiv), + (13, BinOp::URem), + (14, BinOp::SRem), + (15, BinOp::UMulh), + (16, BinOp::SMulh), + ]; + for (tag, op) in bins { + assert_eq!(op as u8, tag, "{op:?}"); + assert_eq!(try_bin_op(tag), Ok(op)); + } + assert!(try_bin_op(17).is_err()); + + let cmps = [ + (0, CmpOp::Eq), + (1, CmpOp::Ne), + (2, CmpOp::Lt), + (3, CmpOp::Le), + (4, CmpOp::Gt), + (5, CmpOp::Ge), + (6, CmpOp::Slt), + (7, CmpOp::Sle), + (8, CmpOp::Sgt), + (9, CmpOp::Sge), + (10, CmpOp::Ult), + (11, CmpOp::Ule), + (12, CmpOp::Ugt), + (13, CmpOp::Uge), + ]; + for (tag, op) in cmps { + assert_eq!(op as u8, tag, "{op:?}"); + assert_eq!(try_cmp_op(tag), Ok(op)); + } + assert!(try_cmp_op(14).is_err()); + + let kinds = [ + (0, ConvertKind::Trunc), + (1, ConvertKind::Zext), + (2, ConvertKind::Sext), + (3, ConvertKind::SiToFp), + (4, ConvertKind::UiToFp), + (5, ConvertKind::FpToSi), + (6, ConvertKind::FpToUi), + (7, ConvertKind::FpResize), + ]; + for (tag, kind) in kinds { + assert_eq!(kind as u8, tag, "{kind:?}"); + assert_eq!(try_convert_kind(tag), Ok(kind)); + } + assert!(try_convert_kind(8).is_err()); + + let atomics = [ + (0, AtomicOp::Add), + (1, AtomicOp::SMin), + (2, AtomicOp::SMax), + (3, AtomicOp::BitAnd), + (4, AtomicOp::BitOr), + (5, AtomicOp::BitXor), + (6, AtomicOp::Exchange), + (7, AtomicOp::CompareExchange), + (8, AtomicOp::UMin), + (9, AtomicOp::UMax), + ]; + for (tag, op) in atomics { + assert_eq!(op as u8, tag, "{op:?}"); + assert_eq!(try_atomic_op(tag), Ok(op)); + } + assert!(try_atomic_op(10).is_err()); + } + + /// Appends one instruction of every signedness-carrying form to a single block. + fn every_operator_function() -> Function { let mut func = Function::new(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, + let i8_t = func.types.intern(TypeKind::Int(IntDesc { bits: 8 })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let i64_t = func.types.intern(TypeKind::Int(IntDesc { bits: 64 })); + let f32_t = func.types.intern(TypeKind::Float(FloatKind::F32)); + let f64_t = func.types.intern(TypeKind::Float(FloatKind::F64)); + let bool_t = func.types.intern(TypeKind::Bool); + let ptr_t = func.types.ptr_global(); + let v16i8 = func.types.intern(TypeKind::Vector(VectorDesc { + len: 16, + elem: i8_t, + })); + let v4i32 = func.types.intern(TypeKind::Vector(VectorDesc { + len: 4, + elem: i32_t, })); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let b = func.append_block_param(entry, i32_t); + let x = func.append_block_param(entry, f32_t); + let y = func.append_block_param(entry, f32_t); + let p = func.append_block_param(entry, ptr_t); + let acc = func.append_block_param(entry, v4i32); + let va = func.append_block_param(entry, v16i8); + let vb = func.append_block_param(entry, v16i8); + let flag = func.append_block_param(entry, bool_t); + let narrow = func.append_block_param(entry, i8_t); + let double = func.append_block_param(entry, f64_t); + + for op in [ + BinOp::Add, + BinOp::Sub, + BinOp::Mul, + BinOp::BitAnd, + BinOp::BitOr, + BinOp::BitXor, + BinOp::Shl, + BinOp::Shr, + BinOp::Sar, + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::UMulh, + BinOp::SMulh, + ] { + func.append_inst(entry, i32_t, Opcode::Arith(Arith { op, lhs: a, rhs: b })); + func.append_arith_imm(entry, i32_t, op, a, 5); + } + for op in [BinOp::Div, BinOp::Rem] { + func.append_inst(entry, f32_t, Opcode::Arith(Arith { op, lhs: x, rhs: y })); + } + for op in [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Slt, + CmpOp::Sle, + CmpOp::Sgt, + CmpOp::Sge, + ] + .into_iter() + .chain([CmpOp::Ult, CmpOp::Ule, CmpOp::Ugt, CmpOp::Uge]) + { + func.append_inst(entry, bool_t, Opcode::Icmp(Compare { op, lhs: a, rhs: b })); + } + for op in [CmpOp::Lt, CmpOp::Le, CmpOp::Gt, CmpOp::Ge] { + func.append_inst(entry, bool_t, Opcode::Icmp(Compare { op, lhs: x, rhs: y })); + } + func.append_convert(entry, i8_t, ConvertKind::Trunc, a); + func.append_convert(entry, i32_t, ConvertKind::Zext, flag); + func.append_convert(entry, i64_t, ConvertKind::Zext, a); + func.append_convert(entry, i64_t, ConvertKind::Sext, a); + func.append_convert(entry, f32_t, ConvertKind::SiToFp, a); + func.append_convert(entry, f32_t, ConvertKind::UiToFp, narrow); + func.append_convert(entry, i32_t, ConvertKind::FpToSi, x); + func.append_convert(entry, i64_t, ConvertKind::FpToUi, double); + func.append_convert(entry, f64_t, ConvertKind::FpResize, x); + func.append_dot(entry, acc, va, vb, true); + func.append_dot(entry, acc, va, vb, false); + for op in [ + AtomicOp::Add, + AtomicOp::SMin, + AtomicOp::SMax, + AtomicOp::UMin, + AtomicOp::UMax, + AtomicOp::BitAnd, + AtomicOp::BitOr, + AtomicOp::BitXor, + AtomicOp::Exchange, + AtomicOp::CompareExchange, + ] { + func.append_atomic_rmw( + entry, + AtomicRmw { + op, + ptr: p, + value: a, + compare: (op == AtomicOp::CompareExchange).then_some(b), + ordering: AtomicOrdering::SeqCst, + scope: AtomicScope::Device, + }, + ); + } + func.set_terminator(entry, Terminator::Ret(Ret::none())); + func + } + + #[test] + fn every_operator_round_trips_field_by_field() { + let func = every_operator_function(); + let decoded = expect_round_trip(&func); + let entry = Block(0); + let (want, got) = (func.block_insts(entry), decoded.block_insts(entry)); + assert_eq!(want.len(), got.len()); + for (w, g) in want.iter().zip(got) { + // Values are numbered by the canonical walk, which here is creation order. + assert_eq!(func.opcode(*w), decoded.opcode(*g)); + let (wt, gt) = ( + func.inst_result(*w).map(|r| func.value_type(r)), + decoded.inst_result(*g).map(|r| decoded.value_type(r)), + ); + let text = |f: &Function, t: Option| t.map(|t| format!("{}", f.types.display(t))); + assert_eq!(text(&func, wt), text(&decoded, gt)); + } + } + + #[test] + fn integer_type_records_have_no_signedness_byte() { + // tag, then the width: `i32` is exactly `[1, 32, 0]`. + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let entry = func.append_block(); + func.append_block_param(entry, i32_t); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + let bytes = encode(&func).unwrap(); + assert_eq!(&bytes[..4], b"VBC3"); + assert_eq!( + &bytes[HEADER_LEN..HEADER_LEN + 4], + &[1, 0, 0, 0], + "one type" + ); + assert_eq!(&bytes[HEADER_LEN + 4..HEADER_LEN + 7], &[1, 32, 0]); + assert!(decode(&bytes).is_ok()); + } + + #[test] + fn a_stream_with_the_previous_magic_is_rejected() { + let func = every_operator_function(); + let mut bytes = encode(&func).unwrap(); + assert!(decode(&bytes).is_ok()); + bytes[..4].copy_from_slice(b"VBC2"); + assert!(decode(&bytes).is_err()); + } + + #[test] + fn an_unknown_convert_kind_or_dot_flag_is_malformed() { + let mut func = Function::new(); + let i8_t = func.types.intern(TypeKind::Int(IntDesc { bits: 8 })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let v4i32 = func.types.intern(TypeKind::Vector(VectorDesc { + len: 4, + elem: i32_t, + })); + let entry = func.append_block(); + let a = func.append_block_param(entry, i8_t); + let acc = func.append_block_param(entry, v4i32); + func.append_convert(entry, i32_t, ConvertKind::Zext, a); + func.append_dot(entry, acc, acc, acc, true); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + let bytes = encode(&func).unwrap(); + assert!(decode(&bytes).is_ok()); + + // The first operand of the convert is value 0, of the dot value 1. + let convert_at = bytes + .windows(6) + .position(|w| w == [OP_CONVERT, ConvertKind::Zext as u8, 0, 0, 0, 0]) + .expect("convert record"); + let mut bad = bytes.clone(); + bad[convert_at + 1] = 8; + assert!(decode(&bad).is_err()); + + let dot_at = bytes + .windows(6) + .position(|w| w == [OP_DOT, DOT_FLAG_SIGNED, 1, 0, 0, 0]) + .expect("dot record"); + let mut bad = bytes.clone(); + bad[dot_at + 1] = DOT_FLAG_SIGNED | 0x80; + assert!(decode(&bad).is_err()); + let mut unsigned = bytes.clone(); + unsigned[dot_at + 1] = 0; + let back = decode(&unsigned).unwrap(); + let last = *back.block_insts(entry).last().unwrap(); + assert!(matches!(back.opcode(last), Opcode::Dot(d) if !d.signed)); + } + + #[test] + fn round_trips_a_compare_exchange_and_its_compare_operand() { + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let ptr_t = func.types.ptr_global(); let entry = func.append_block(); let p = func.append_block_param(entry, ptr_t); @@ -2524,10 +3015,7 @@ mod tests { #[test] fn an_unknown_atomic_selector_byte_is_rejected() { let mut func = Function::new(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let ptr_t = func.types.ptr_global(); let entry = func.append_block(); let p = func.append_block_param(entry, ptr_t); @@ -2631,17 +3119,14 @@ mod tests { #[test] fn rejects_truncated_bitcode() { - assert!(decode(b"VBC1\x01").is_err()); + assert!(decode(b"VBC3\x01").is_err()); assert!(decode(b"nope").is_err()); } #[test] fn round_trips_va_start_va_arg_va_end_through_bitcode() { let mut func = Function::new(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let ptr_t = func.types.ptr_global(); let entry = func.append_block(); let list = func.append_block_param(entry, ptr_t); @@ -2719,17 +3204,17 @@ mod tests { // An unknown arith operator byte, no such BinOp. let bad_arith = bconcat![ TEST_HEADER, - [1, 0, 0, 0], // type_count = 1 - [1, 0, 32, 0], // int, signed, 32 bits - [0, 0, 0, 0], // sym_count = 0 - [1, 0, 0, 0], // block_count = 1 - [1, 0, 0, 0], // block 0 param_count = 1 - [0, 0, 0, 0], // param 0 type 0 - [1, 0, 0, 0], // inst_count = 1 - [1], // has_result = 1 - [0, 0, 0, 0], // result type 0 - [2], // tag = op_arith - [0xff], // operator byte: no such BinOp + [1, 0, 0, 0], // type_count = 1 + [1, 32, 0], // int, 32 bits + [0, 0, 0, 0], // sym_count = 0 + [1, 0, 0, 0], // block_count = 1 + [1, 0, 0, 0], // block 0 param_count = 1 + [0, 0, 0, 0], // param 0 type 0 + [1, 0, 0, 0], // inst_count = 1 + [1], // has_result = 1 + [0, 0, 0, 0], // result type 0 + [2], // tag = op_arith + [0xff], // operator byte: no such BinOp ]; assert!(decode(&bad_arith[..]).is_err()); @@ -2812,10 +3297,7 @@ mod tests { let mut func = Function::new(); let s = func.types.intern(TypeKind::Ptr(AddressSpace::Shared)); let g = func.types.ptr_global(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let b = func.append_block(); let ps = func.append_block_param(b, s); let pg = func.append_block_param(b, g); @@ -2838,10 +3320,7 @@ mod tests { #[test] fn a_two_value_ret_round_trips_through_bitcode() { let mut func = Function::new(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let entry = func.append_block(); let a = func.append_block_param(entry, i32_t); let b = func.append_block_param(entry, i32_t); @@ -2859,4 +3338,163 @@ mod tests { assert_eq!(params[1], decoded_ret.values[1]); let _ = expect_round_trip(&func); } + + fn all_mem_flags() -> Vec { + let orderings = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Acquire), + Some(AtomicOrdering::Release), + Some(AtomicOrdering::AcqRel), + Some(AtomicOrdering::SeqCst), + ]; + let endians = [Endianness::Native, Endianness::Little, Endianness::Big]; + let mut out = Vec::new(); + for volatile in [false, true] { + for align in [0u32, 1, 2, 8, 4096, 1 << 31] { + for ordering in orderings { + for endian in endians { + out.push(MemFlags { + volatile, + align, + ordering, + endian, + }); + } + } + } + } + out + } + + #[test] + fn loads_and_stores_with_every_mem_flag_round_trip_through_bitcode() { + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let mut expected = Vec::new(); + let mut last = None; + for mem in all_mem_flags() { + let v = func.append_load(entry, i32_t, p, mem); + func.append_store_mem(entry, v, p, mem); + expected.push(mem); + expected.push(mem); + last = Some(v); + } + func.set_terminator(entry, Terminator::Ret(Ret::one(last.unwrap()))); + + let decoded = expect_round_trip(&func); + let got: Vec = decoded + .block_insts(entry) + .iter() + .map(|&i| match decoded.opcode(i) { + Opcode::Load(l) => l.mem, + Opcode::Store(st) => st.mem, + _ => unreachable!(), + }) + .collect(); + assert_eq!(got, expected); + } + + #[test] + fn mem_flags_wire_layout_is_pinned() { + let encoded = |mem: MemFlags| { + let mut w = Writer::new(); + write_mem_flags(&mut w, &mem).unwrap(); + w.bytes + }; + assert_eq!(encoded(MemFlags::new()), [0, 0]); + assert_eq!( + encoded(MemFlags { + volatile: true, + align: 8, + ordering: Some(AtomicOrdering::SeqCst), + endian: Endianness::Big, + }), + [1 | (2 << 1) | (5 << 3), 4] + ); + assert_eq!( + encoded(MemFlags { + endian: Endianness::Little, + ordering: Some(AtomicOrdering::Relaxed), + align: 1, + ..MemFlags::new() + }), + [(1 << 1) | (1 << 3), 1] + ); + } + + #[test] + fn malformed_mem_flags_are_rejected() { + let read = |bytes: &[u8]| read_mem_flags(&mut Reader { bytes, pos: 0 }); + assert!(read(&[0, 0]).is_ok()); + assert!(read(&[3 << 1, 0]).is_err(), "endian selector 3"); + assert!(read(&[6 << 3, 0]).is_err(), "ordering selector 6"); + assert!(read(&[7 << 3, 0]).is_err(), "ordering selector 7"); + assert!(read(&[1 << 6, 0]).is_err(), "reserved flag bit"); + assert!(read(&[0, 33]).is_err(), "align code past 2^31"); + assert!(read(&[0]).is_err(), "truncated"); + let mut w = Writer::new(); + let odd = MemFlags { + align: 3, + ..MemFlags::new() + }; + assert!(write_mem_flags(&mut w, &odd).is_err()); + } + + #[test] + fn a_non_power_of_two_align_fails_to_encode() { + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let v = func.append_load( + entry, + i32_t, + p, + MemFlags { + align: 6, + ..MemFlags::new() + }, + ); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + assert!(encode(&func).is_err()); + } + + #[test] + fn intrinsics_round_trip_through_bitcode() { + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let b = func.append_block_param(entry, i32_t); + let valued = func.append_intrinsic(entry, i32_t, "aarch64.aese", &[a, b], -7, true); + let pure = func.append_intrinsic(entry, i32_t, "x86.bswap", &[valued], 0, false); + func.append_intrinsic_void(entry, "arm.dmb", &[], 15, true); + func.append_intrinsic_void(entry, "nop.pure", &[a], i64::MIN, false); + func.set_terminator(entry, Terminator::Ret(Ret::one(pure))); + + let decoded = expect_round_trip(&func); + let insts = decoded.block_insts(entry).to_vec(); + assert_eq!(insts.len(), 4); + let get = |n: usize| match decoded.opcode(insts[n]) { + Opcode::Intrinsic(i) => i, + _ => panic!("expected intrinsic"), + }; + let params = decoded.block_params(entry).to_vec(); + let v0 = decoded.inst_result(insts[0]).unwrap(); + assert_eq!(decoded.symbol_name(get(0).symbol), "aarch64.aese"); + assert_eq!(decoded.value_list(get(0).args), ¶ms[..]); + assert_eq!((get(0).imm, get(0).side_effects), (-7, true)); + assert_eq!(decoded.value_list(get(1).args), &[v0]); + assert!(!get(1).side_effects && decoded.inst_result(insts[1]).is_some()); + assert_eq!(decoded.symbol_name(get(2).symbol), "arm.dmb"); + assert!(decoded.inst_result(insts[2]).is_none() && get(2).imm == 15); + assert_eq!(get(3).imm, i64::MIN); + assert!(!get(3).side_effects); + assert_eq!(decoded.value_list(get(3).args), ¶ms[..1]); + } } diff --git a/src/ir/src/builder.rs b/src/ir/src/builder.rs index 6a9f0aa..efa9e9f 100644 --- a/src/ir/src/builder.rs +++ b/src/ir/src/builder.rs @@ -1,6 +1,6 @@ extern crate alloc; -use super::function::{BinOp, Block, CmpOp, EdgeDesc, Function, Value}; +use super::function::{BinOp, Block, CmpOp, ConvertKind, EdgeDesc, Function, MemFlags, Value}; use super::types::Type; pub struct Builder<'a> { @@ -44,6 +44,11 @@ impl<'a> Builder<'a> { ) } + /// `convert , value`; see [`ConvertKind`] for the legal type pairs. + pub fn convert(&mut self, ty: Type, kind: ConvertKind, value: Value) -> Value { + self.func.append_convert(self.block, ty, kind, value) + } + pub fn select(&mut self, cond: Value, then_value: Value, else_value: Value) -> Value { let ty = self.func.value_type(then_value); self.func.append_inst( @@ -76,20 +81,38 @@ impl<'a> Builder<'a> { } pub fn load(&mut self, ty: Type, ptr: Value) -> Value { - self.func.append_inst( - self.block, - ty, - super::function::Opcode::Load(super::function::Load { - ptr, - volatile: false, - }), - ) + self.load_mem(ty, ptr, MemFlags::new()) + } + + pub fn load_mem(&mut self, ty: Type, ptr: Value, mem: MemFlags) -> Value { + self.func.append_load(self.block, ty, ptr, mem) } pub fn store(&mut self, value: Value, ptr: Value) { self.func.append_store(self.block, value, ptr); } + pub fn store_mem(&mut self, value: Value, ptr: Value, mem: MemFlags) { + self.func.append_store_mem(self.block, value, ptr, mem); + } + + pub fn intrinsic( + &mut self, + ty: Type, + name: &str, + args: &[Value], + imm: i64, + side_effects: bool, + ) -> Value { + self.func + .append_intrinsic(self.block, ty, name, args, imm, side_effects) + } + + pub fn intrinsic_void(&mut self, name: &str, args: &[Value], imm: i64, side_effects: bool) { + self.func + .append_intrinsic_void(self.block, name, args, imm, side_effects); + } + pub fn ret(&mut self, value: Option) { let r = match value { Some(v) => super::function::Ret::one(v), @@ -122,10 +145,7 @@ mod tests { use super::*; fn i32t(f: &mut Function) -> Type { - f.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })) + f.types.intern(TypeKind::Int(IntDesc { bits: 32 })) } #[test] @@ -173,10 +193,29 @@ mod tests { let a = func.append_block_param(entry, i32_t); let b = func.append_block_param(entry, i32_t); let mut bld = Builder::new(&mut func, entry); - let c = bld.icmp(CmpOp::Gt, a, b); + let c = bld.icmp(CmpOp::Sgt, a, b); assert_eq!(bool_t, func.value_type(c)); } + #[test] + fn convert_takes_the_result_type_and_kind() { + let mut func = Function::new(); + let i32_t = i32t(&mut func); + let i64_t = func.types.intern(TypeKind::Int(IntDesc { bits: 64 })); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let mut bld = Builder::new(&mut func, entry); + let wide = bld.convert(i64_t, ConvertKind::Sext, a); + assert_eq!(i64_t, func.value_type(wide)); + match func.opcode(func.defining_inst(wide).unwrap()) { + super::super::function::Opcode::Convert(c) => { + assert_eq!(a, c.value); + assert_eq!(ConvertKind::Sext, c.kind); + } + _ => panic!("expected convert"), + } + } + #[test] fn struct_construction_and_extraction() { let mut func = Function::new(); diff --git a/src/ir/src/dialect/attrs.rs b/src/ir/src/dialect/attrs.rs deleted file mode 100644 index 2e4eada..0000000 --- a/src/ir/src/dialect/attrs.rs +++ /dev/null @@ -1,726 +0,0 @@ -//! Operation payload attributes for the `volt` dialect. -//! -//! Small enums reuse the existing volt definitions (from -//! [`function`](crate::function), [`low_float`](crate::low_float), -//! [`nvfp4`](crate::nvfp4), and [`attribute`](crate::attribute)); the -//! structs below wrap them as pliron attributes with keyword syntax. - -use alloc::string::{String, ToString}; -use alloc::vec::Vec; - -use pliron::{ - combine::{ - Parser, choice, - parser::{ - char::{spaces, string}, - token::{satisfy, token}, - }, - }, - context::Context, - derive::pliron_attr, - irfmt::parsers::int_parser, - parsable::{Parsable, ParseResult, StateStream}, - printable::{self, Printable}, -}; - -use super::super::attribute::{AttrValue, Attribute, Custom, Endianness}; -use super::super::function::{ - AtomicOp, AtomicOrdering, AtomicScope, BarrierScope, BinOp, CmpOp, InputSigns, MatMulQuantOut, - MatMulType, RetPiece, UnaryOp, -}; -use super::super::low_float::Format as LowFloatFormat; -use super::super::nvfp4::ScaleApplication; - -/// Parse one of several keywords. -/// Parse one of several keywords. -fn keyword<'a, T: Copy>( - state_stream: &mut StateStream<'a>, - options: &[(&'static str, T)], -) -> ParseResult<'a, T> { - let mut parsers = Vec::new(); - for (word, value) in options { - parsers.push(string(*word).map(move |_| *value)); - } - spaces() - .with(choice(&mut parsers[..])) - .parse_stream(state_stream) - .into() -} - -macro_rules! keyword_attr { - ($attr:ident, $inner:ty, $name:literal, [$($word:literal => $variant:path),* $(,)?]) => { - #[doc = concat!("A `", $name, "` attribute.")] - #[pliron_attr(name = $name, verifier = "succ")] - #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] - pub struct $attr(pub $inner); - - impl $attr { - /// Wrap a value. - pub fn new(value: $inner) -> Self { - Self(value) - } - - /// The wrapped value. - pub fn value(&self) -> $inner { - self.0 - } - } - - impl Printable for $attr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let word = match self.0 { - $($variant => $word,)* - }; - f.write_str(word) - } - } - - impl Parsable for $attr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - keyword(state_stream, &[$(($word, $variant),)*]) - .map(|(val, rest)| (Self(val), rest)) - } - } - }; -} - -keyword_attr!( - BinOpAttr, - BinOp, - "volt.bin_op", - [ - "add" => BinOp::Add, "sub" => BinOp::Sub, "mul" => BinOp::Mul, - "div" => BinOp::Div, "rem" => BinOp::Rem, "and" => BinOp::BitAnd, - "or" => BinOp::BitOr, "xor" => BinOp::BitXor, "shl" => BinOp::Shl, - "shr" => BinOp::Shr, "mulh" => BinOp::Mulh, - ] -); - -keyword_attr!( - CmpOpAttr, - CmpOp, - "volt.cmp_op", - [ - "eq" => CmpOp::Eq, "ne" => CmpOp::Ne, "lt" => CmpOp::Lt, - "le" => CmpOp::Le, "gt" => CmpOp::Gt, "ge" => CmpOp::Ge, - ] -); - -keyword_attr!( - UnaryOpAttr, - UnaryOp, - "volt.unary_op", - [ - "reinterpret" => UnaryOp::Reinterpret, "sqrt" => UnaryOp::Sqrt, - "ceil" => UnaryOp::Ceil, "floor" => UnaryOp::Floor, - "trunc" => UnaryOp::Trunc, "nearest" => UnaryOp::Nearest, - ] -); - -keyword_attr!( - BarrierAttr, - BarrierScope, - "volt.barrier_scope", - [ - "workgroup" => BarrierScope::Workgroup, - "subgroup" => BarrierScope::Subgroup, - ] -); - -keyword_attr!( - AtomicOpAttr, - AtomicOp, - "volt.atomic_op", - [ - "add" => AtomicOp::Add, "min" => AtomicOp::Min, - "max" => AtomicOp::Max, "and" => AtomicOp::BitAnd, - "or" => AtomicOp::BitOr, "xor" => AtomicOp::BitXor, - "exchange" => AtomicOp::Exchange, - "compare_exchange" => AtomicOp::CompareExchange, - ] -); - -keyword_attr!( - AtomicOrderingAttr, - AtomicOrdering, - "volt.atomic_ordering", - [ - "relaxed" => AtomicOrdering::Relaxed, - "acquire" => AtomicOrdering::Acquire, - "release" => AtomicOrdering::Release, - "acq_rel" => AtomicOrdering::AcqRel, - "seq_cst" => AtomicOrdering::SeqCst, - ] -); - -keyword_attr!( - AtomicScopeAttr, - AtomicScope, - "volt.atomic_scope", - [ - "workgroup" => AtomicScope::Workgroup, - "device" => AtomicScope::Device, - "system" => AtomicScope::System, - ] -); - -keyword_attr!( - LowFloatAttr, - LowFloatFormat, - "volt.low_float_format", - [ - "bf16" => LowFloatFormat::Bf16, - "f8e4m3" => LowFloatFormat::F8E4M3, - "f8e5m2" => LowFloatFormat::F8E5M2, - ] -); - -keyword_attr!( - ScaleAppAttr, - ScaleApplication, - "volt.scale_application", - [ - "multiply" => ScaleApplication::Multiply, - "divide" => ScaleApplication::Divide, - ] -); - -keyword_attr!( - MatMulTypeAttr, - MatMulType, - "volt.matmul_dtype", - [ - "fp32" => MatMulType::Fp32, "fp16" => MatMulType::Fp16, - "int8" => MatMulType::Int8, "uint8" => MatMulType::Uint8, - ] -); - -keyword_attr!( - MatMulQuantOutAttr, - MatMulQuantOut, - "volt.matmul_quant_out", - [ - "i8" => MatMulQuantOut::I8, - "u8" => MatMulQuantOut::U8, - ] -); - -/// A generic signed integer attribute: immediates, indices, counts. -#[pliron_attr(name = "volt.imm", verifier = "succ")] -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct ImmAttr(pub i64); - -impl ImmAttr { - /// Wrap a value. - pub fn new(value: i64) -> Self { - Self(value) - } - - /// The wrapped value. - pub fn value(&self) -> i64 { - self.0 - } -} - -impl Printable for ImmAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl Parsable for ImmAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(int_parser::()) - .map(Self) - .parse_stream(state_stream) - .into() - } -} - -/// Raw bits of an `f64` constant, printed as decimal. -#[pliron_attr(name = "volt.fbits", verifier = "succ")] -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct FloatBitsAttr(pub u64); - -impl FloatBitsAttr { - /// Wrap an `f64` constant. - pub fn new(value: f64) -> Self { - Self(value.to_bits()) - } - - /// The constant value. - pub fn value(&self) -> f64 { - f64::from_bits(self.0) - } -} - -impl Printable for FloatBitsAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl Parsable for FloatBitsAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(int_parser::()) - .map(Self) - .parse_stream(state_stream) - .into() - } -} - -/// Raw bits of an `f128` constant, printed as decimal. -#[pliron_attr(name = "volt.f128bits", verifier = "succ")] -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct F128BitsAttr(pub u128); - -impl F128BitsAttr { - /// Wrap a value. - pub fn new(value: u128) -> Self { - Self(value) - } - - /// The wrapped value. - pub fn value(&self) -> u128 { - self.0 - } -} - -impl Printable for F128BitsAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl Parsable for F128BitsAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(int_parser::()) - .map(Self) - .parse_stream(state_stream) - .into() - } -} - -/// Owned matrix-multiply configuration (mirrors [`MatMul`](crate::function::MatMul)). -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct MatMulDesc { - /// Rows of A / C. - pub m: u16, - /// Columns of B / C. - pub n: u16, - /// Reduction dimension. - pub k: u16, - /// Element flavor. - pub dtype: MatMulType, - /// Accumulate into C instead of overwriting. - pub accumulate: bool, - /// Embedded (in-register) operand layout. - pub embedded: bool, - /// Optional quantization parameters. - pub quant: Option, - /// Optional operand signedness overrides. - pub input_signs: Option, -} - -/// Owned quantization parameters (mirrors [`MatMulQuant`](crate::function::MatMulQuant)). -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct QuantDesc { - /// Single shared scale, if any. - pub scale_scalar: Option, - /// Per-column scales, if any. - pub scale_per_column: Option>, - /// Apply ReLU to the output. - pub relu: bool, - /// Output flavor. - pub out: MatMulQuantOut, - /// Per-output bias, if any. - pub bias: Option>, - /// Zero point. - pub zero_point: i32, -} - -/// A `volt.matmul_config` attribute. -#[pliron_attr(name = "volt.matmul_config", verifier = "succ")] -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct MatMulAttr(pub MatMulDesc); - -impl MatMulAttr { - /// Wrap a descriptor. - pub fn new(desc: MatMulDesc) -> Self { - Self(desc) - } - - /// The descriptor. - pub fn desc(&self) -> &MatMulDesc { - &self.0 - } -} - -impl Printable for MatMulAttr { - fn fmt( - &self, - ctx: &Context, - state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let d = &self.0; - write!( - f, - "m={} n={} k={} dtype={} ", - d.m, - d.n, - d.k, - MatMulTypeAttr(d.dtype).print(ctx, state) - )?; - if d.accumulate { - write!(f, "acc ")?; - } - if d.embedded { - write!(f, "embedded ")?; - } - if let Some(q) = &d.quant { - write!(f, "quant(")?; - if let Some(s) = q.scale_scalar { - write!(f, "scale={s} ")?; - } - if let Some(scales) = &q.scale_per_column { - write!(f, "scales=[")?; - for (i, s) in scales.iter().enumerate() { - if i != 0 { - write!(f, ", ")?; - } - write!(f, "{s}")?; - } - write!(f, "] ")?; - } - if q.relu { - write!(f, "relu ")?; - } - write!(f, "out={} ", MatMulQuantOutAttr(q.out).print(ctx, state))?; - if let Some(bias) = &q.bias { - write!(f, "bias=[")?; - for (i, b) in bias.iter().enumerate() { - if i != 0 { - write!(f, ", ")?; - } - write!(f, "{b}")?; - } - write!(f, "] ")?; - } - write!(f, "zp={} ) ", q.zero_point)?; - } - if let Some(signs) = &d.input_signs { - write!( - f, - "signs(a_unsigned={} b_unsigned={}) ", - signs.a_unsigned, signs.b_unsigned - )?; - } - Ok(()) - } -} - -impl Parsable for MatMulAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - _state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - // TODO: picks up in the parser port. - unimplemented!("matmul config parsing") - } -} - -/// Call ABI details: variadic shape, struct-return, multi-register returns. -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct CallInfo { - /// Variadic call. - pub is_variadic: bool, - /// Number of fixed arguments (variadic calls). - pub num_fixed: u32, - /// Struct-return (first argument is the return slot). - pub sret: bool, - /// Number of registers used by the return value. - pub ret_regs: u8, - /// How the return value is split across registers. - pub ret_pieces: Vec, -} - -impl Default for CallInfo { - fn default() -> Self { - Self { - is_variadic: false, - num_fixed: 0, - sret: false, - ret_regs: 0, - ret_pieces: Vec::new(), - } - } -} - -/// A `volt.call_info` attribute. -#[pliron_attr(name = "volt.call_info", verifier = "succ")] -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct CallInfoAttr(pub CallInfo); - -impl CallInfoAttr { - /// Wrap call info. - pub fn new(info: CallInfo) -> Self { - Self(info) - } - - /// The call info. - pub fn info(&self) -> &CallInfo { - &self.0 - } -} - -impl Printable for CallInfoAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let c = &self.0; - write!( - f, - "variadic={} fixed={} sret={} regs={} pieces=[", - c.is_variadic, c.num_fixed, c.sret, c.ret_regs - )?; - for (i, p) in c.ret_pieces.iter().enumerate() { - if i != 0 { - write!(f, ", ")?; - } - write!(f, "{}:{}:{}", p.fp as u8, p.offset, p.bytes)?; - } - write!(f, "]") - } -} - -impl Parsable for CallInfoAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - _state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - // TODO: picks up in the parser port. - unimplemented!("call info parsing") - } -} - -/// Function-level flags: variadic shape, struct-return, linkage. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)] -pub struct FuncInfo { - /// Variadic function. - pub is_variadic: bool, - /// Number of fixed parameters (variadic functions). - pub num_fixed_params: u32, - /// Struct-return (first parameter is the return slot). - pub sret: bool, - /// Local (non-exported) linkage. - pub is_local: bool, -} - -/// A `volt.func_info` attribute. -#[pliron_attr(name = "volt.func_info", verifier = "succ")] -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)] -pub struct FuncInfoAttr(pub FuncInfo); - -impl FuncInfoAttr { - /// Wrap function info. - pub fn new(info: FuncInfo) -> Self { - Self(info) - } - - /// The function info. - pub fn info(&self) -> FuncInfo { - self.0 - } -} - -impl Printable for FuncInfoAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let i = &self.0; - write!( - f, - "variadic={} fixed={} sret={} local={}", - i.is_variadic, i.num_fixed_params, i.sret, i.is_local - ) - } -} - -impl Parsable for FuncInfoAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - _state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - // TODO: picks up in the parser port. - unimplemented!("func info parsing") - } -} - -/// A `volt.generic_attr` wrapping any legacy [`Attribute`], so existing -/// attribute flows (inline, align, endian, custom, ...) keep working. -/// Prints with the legacy syntax (`inline`, `align(16)`, `ns.key = 1`). -#[pliron_attr(name = "volt.generic_attr", verifier = "succ")] -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct VoltAttr(pub Attribute); - -impl VoltAttr { - /// Wrap a legacy attribute. - pub fn new(attr: Attribute) -> Self { - Self(attr) - } - - /// The wrapped attribute. - pub fn attr(&self) -> &Attribute { - &self.0 - } -} - -impl Printable for VoltAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - match &self.0 { - Attribute::Inline => f.write_str("inline"), - Attribute::Noreturn => f.write_str("noreturn"), - Attribute::Cold => f.write_str("cold"), - Attribute::Align(a) => write!(f, "align({a})"), - Attribute::Endian(e) => { - let s = match e { - Endianness::Little => "little", - Endianness::Big => "big", - Endianness::Native => "native", - }; - write!(f, "endian({s})") - } - Attribute::Custom(c) => { - write!(f, "{}.{}", c.namespace, c.key)?; - match &c.value { - AttrValue::Flag => Ok(()), - AttrValue::Int(i) => write!(f, " = {i}"), - AttrValue::Str(s) => write!(f, " = \"{s}\""), - } - } - } - } -} - -fn quoted_string<'a>(state_stream: &mut StateStream<'a>) -> ParseResult<'a, String> { - token('"') - .with(pliron::combine::many(satisfy(|c: char| c != '"'))) - .skip(token('"')) - .map(|chars: Vec| chars.into_iter().collect::()) - .parse_stream(state_stream) - .into() -} - -impl Parsable for VoltAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - _state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - // TODO: picks up in the parser port. - unimplemented!("generic attribute parsing") - } -} - -/// Parse a legacy dotted-path custom attribute name (`a.b.key`). -#[allow(dead_code)] -pub fn parse_custom_path<'a>(state_stream: &mut StateStream<'a>) -> ParseResult<'a, String> { - ( - pliron::combine::many1(satisfy(|c: char| c.is_ascii_alphanumeric() || c == '_')) - .map(|w: Vec| w.into_iter().collect::()), - pliron::combine::many(token('.').with(pliron::combine::many1(satisfy(|c: char| { - c.is_ascii_alphanumeric() || c == '_' - })))), - ) - .map(|(first, rest): (String, Vec>)| { - let mut path = first; - for seg in rest { - path.push('.'); - path.extend(seg); - } - path - }) - .parse_stream(state_stream) - .into() -} - -#[allow(dead_code)] -fn _use_quoted(state: &mut StateStream<'_>) { - let _ = quoted_string(state); - let _ = parse_custom_path(state); -} - -impl Custom { - /// Split a dotted path into namespace + key at the last dot. - #[allow(dead_code)] - pub fn split_path(path: String) -> Option { - let dot = path.rfind('.')?; - Some(Custom { - namespace: path[..dot].to_string(), - key: path[dot + 1..].to_string(), - value: AttrValue::Flag, - }) - } -} diff --git a/src/ir/src/dialect/func.rs b/src/ir/src/dialect/func.rs deleted file mode 100644 index 5e1daac..0000000 --- a/src/ir/src/dialect/func.rs +++ /dev/null @@ -1,1929 +0,0 @@ -//! Function-level container, builder, and legacy IR bridge over the `volt` dialect. - -extern crate alloc; - -use alloc::string::{String, ToString}; -use alloc::vec::Vec; -use core::fmt; -use std::collections::HashMap; - -use pliron::{ - basic_block::BasicBlock, - builtin::{ - op_interfaces::{OneRegionInterface, SymbolOpInterface}, - ops::FuncOp, - types::FunctionType, - }, - common_traits::Verify, - context::{Context, Ptr}, - identifier::Identifier, - linked_list::ContainsLinkedList, - op::Op, - operation::Operation, - printable::Printable, - region::Region, - r#type::{TypeHandle, Typed}, - value::Value, -}; - -use super::attrs::{CallInfo, FuncInfo, FuncInfoAttr, MatMulDesc, QuantDesc}; -use super::ops::{ - AllocaOp, ArithImmOp, ArithOp, AtomicRmwOp, BarrierOp, CallIndirectOp, CallOp, CondBrOp, - ConstF128Op, ConstFloatOp, ConstIntOp, ConvertOp, DecodeLowFloatOp, DequantizeNvfp4Op, DotOp, - EncodeLowFloatOp, ExtractOp, GlobalAddrOp, IcmpOp, JumpOp, LoadOp, MatMulOp, PrefetchOp, - QuantizeNvfp4Op, ReduceOp, RetOp, SelectOp, SplatOp, StoreOp, StructNewOp, UnaryOp, VaArgOp, - VaEndOp, VaStartOp, -}; -use super::types::{ - ArrayType, BoolType, FloatType, IntType, PtrType, SliceType, StructType, VectorType, VoltKind, - describe, -}; -use crate::function::{ - AtomicOp, AtomicOrdering, AtomicScope, BarrierScope, BinOp, CmpOp, RetPiece, -}; -use crate::low_float::Format as LowFloatFormat; -use crate::nvfp4::ScaleApplication; - -/// Error encountered during conversion between pliron IR and legacy IR. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ConvertError { - UnsupportedOp(String), - UnsupportedType(String), - MissingValue, - MissingBlock, - InvalidSignature, -} - -impl fmt::Display for ConvertError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::UnsupportedOp(op) => write!(f, "unsupported operation: {op}"), - Self::UnsupportedType(ty) => write!(f, "unsupported type: {ty}"), - Self::MissingValue => f.write_str("missing SSA value during lowering"), - Self::MissingBlock => f.write_str("missing block target during lowering"), - Self::InvalidSignature => f.write_str("invalid function signature"), - } - } -} - -impl core::error::Error for ConvertError {} - -/// A function container over the Volt dialect. -pub struct Function { - pub ctx: Context, - pub func: FuncOp, -} - -impl Function { - /// Create a new function with an owned context, entry block, and typed arguments. - pub fn new(name: &str, param_types: Vec, return_types: Vec) -> Self { - let mut ctx = Context::new(); - let func_ty = FunctionType::get(&mut ctx, param_types, return_types); - let id = Identifier::try_from(name).expect("valid function name"); - let func = FuncOp::new(&mut ctx, id, func_ty); - Self { ctx, func } - } - - /// Create a new function within an existing context. - pub fn with_context( - mut ctx: Context, - name: &str, - param_types: Vec, - return_types: Vec, - ) -> Self { - let func_ty = FunctionType::get(&mut ctx, param_types, return_types); - let id = Identifier::try_from(name).expect("valid function name"); - let func = FuncOp::new(&mut ctx, id, func_ty); - Self { ctx, func } - } - - /// Wrap an existing [`FuncOp`] and its context. - pub fn from_func_op(ctx: Context, func: FuncOp) -> Self { - Self { ctx, func } - } - - /// Reference to the underlying context. - pub fn context(&self) -> &Context { - &self.ctx - } - - /// Mutable reference to the underlying context. - pub fn context_mut(&mut self) -> &mut Context { - &mut self.ctx - } - - /// The function's [`FuncOp`]. - pub fn func_op(&self) -> FuncOp { - self.func - } - - /// The function symbol name. - pub fn name(&self) -> Identifier { - self.func.get_symbol_name(&self.ctx) - } - - /// The entry basic block. - pub fn entry_block(&self) -> Ptr { - self.func.get_entry_block(&self.ctx) - } - - /// The single region of the function. - pub fn region(&self) -> Ptr { - self.func.get_region(&self.ctx) - } - - /// All basic blocks in the function in order. - pub fn blocks(&self) -> Vec> { - self.region().deref(&self.ctx).iter(&self.ctx).collect() - } - - /// Append a new basic block to the function with optional label and arguments. - pub fn append_block( - &mut self, - label: Option<&str>, - arg_types: Vec, - ) -> Ptr { - let id = label.map(|s| Identifier::try_from(s).expect("valid block label")); - let bb = BasicBlock::new(&mut self.ctx, id, arg_types); - let r = self.region(); - bb.insert_at_back(r, &mut self.ctx); - bb - } - - /// Obtain an instruction builder positioned at the end of the given block. - pub fn builder(&mut self, block: Ptr) -> Builder<'_> { - Builder::new(&mut self.ctx, block) - } - - /// Set function-level information flags. - pub fn set_func_info(&mut self, info: FuncInfo) { - let key = pliron::ident!("volt_func_info"); - self.func - .get_operation() - .deref_mut(&mut self.ctx) - .attributes - .set(key, FuncInfoAttr::new(info)); - } - - /// Retrieve function-level information flags, if present. - pub fn func_info(&self) -> Option { - let key = pliron::ident!("volt_func_info"); - self.func - .get_operation() - .deref(&self.ctx) - .attributes - .get::(&key) - .map(|a| a.info()) - } - - /// Verify the function. - pub fn verify(&self) -> Result<(), pliron::result::Error> { - self.func.get_operation().deref(&self.ctx).verify(&self.ctx) - } - - /// Print the function to a string. - pub fn print(&self) -> String { - alloc::format!( - "{}", - self.func - .get_operation() - .deref(&self.ctx) - .print(&self.ctx, &Default::default()) - ) - } - - /// Convert a legacy [`crate::function::Function`] into a pliron [`Function`]. - pub fn from_legacy(legacy: &crate::function::Function) -> Self { - let mut ctx = Context::new(); - - // Convert all types up-front. - let type_map: Vec = (0..legacy.types.count()) - .map(|i| { - let ty = crate::types::Type(i as u32); - convert_legacy_type_to_pliron(&mut ctx, &legacy.types, ty) - }) - .collect(); - - // Determine input types and return types from the entry block and return statements. - let (param_types, ret_types) = if legacy.block_count() > 0 { - let entry_params = legacy.block_params(crate::function::Block(0)); - let params = entry_params - .iter() - .map(|&p| type_map[legacy.value_type(p).0 as usize]) - .collect(); - - // Find first return terminator to determine return types. - let mut rets = Vec::new(); - for b in 0..legacy.block_count() { - if let Some(crate::function::Terminator::Ret(r)) = - legacy.terminator(crate::function::Block(b as u32)) - { - rets = r - .slice() - .iter() - .map(|&v| type_map[legacy.value_type(v).0 as usize]) - .collect(); - break; - } - } - (params, rets) - } else { - (Vec::new(), Vec::new()) - }; - - let func_ty = FunctionType::get(&mut ctx, param_types, ret_types); - let func = FuncOp::new(&mut ctx, pliron::ident!("func"), func_ty); - let region = func.get_region(&ctx); - - if legacy.block_count() == 0 { - let mut res = Self { ctx, func }; - res.set_func_info(FuncInfo { - is_variadic: legacy.is_variadic, - num_fixed_params: legacy.num_fixed_params, - sret: legacy.sret, - is_local: legacy.is_local, - }); - return res; - } - - // Map blocks: block 0 is the entry block, blocks 1..N are created. - let mut blocks = Vec::with_capacity(legacy.block_count()); - let entry_bb = func.get_entry_block(&ctx); - blocks.push(entry_bb); - - for b in 1..legacy.block_count() { - let param_tys: Vec = legacy - .block_params(crate::function::Block(b as u32)) - .iter() - .map(|&p| type_map[legacy.value_type(p).0 as usize]) - .collect(); - let label = Identifier::try_from(alloc::format!("block{b}")).expect("valid block name"); - let bb = BasicBlock::new(&mut ctx, Some(label), param_tys); - bb.insert_at_back(region, &mut ctx); - blocks.push(bb); - } - - // Map values (indexed by legacy Value index). - let mut val_map: Vec> = alloc::vec![None; legacy.value_count()]; - - // Map entry block parameters. - for (i, &p) in legacy - .block_params(crate::function::Block(0)) - .iter() - .enumerate() - { - val_map[p.index()] = Some(entry_bb.deref(&ctx).get_argument(i)); - } - - // Map other block parameters. - for b in 1..legacy.block_count() { - let bb = blocks[b]; - for (i, &p) in legacy - .block_params(crate::function::Block(b as u32)) - .iter() - .enumerate() - { - val_map[p.index()] = Some(bb.deref(&ctx).get_argument(i)); - } - } - - // Translate instructions per block. - for b in 0..legacy.block_count() { - let bb = blocks[b]; - let legacy_b = crate::function::Block(b as u32); - - for inst in legacy.block_insts(legacy_b) { - let op = legacy.opcode(*inst); - let pliron_ty = legacy - .inst_result(*inst) - .map(|r| type_map[legacy.value_type(r).0 as usize]) - .unwrap_or_else(|| BoolType::get_type(&ctx)); - - match op { - crate::function::Opcode::Iconst(val) => { - let op = ConstIntOp::new(&mut ctx, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Fconst(val) => { - let op = ConstFloatOp::new(&mut ctx, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Fconst128(val) => { - let op = ConstF128Op::new(&mut ctx, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Arith(a) => { - let lhs = val_map[a.lhs.index()].expect("operand"); - let rhs = val_map[a.rhs.index()].expect("operand"); - let op = ArithOp::new(&mut ctx, a.op, lhs, rhs, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::ArithImm(a) => { - let lhs = val_map[a.lhs.index()].expect("operand"); - let op = ArithImmOp::new(&mut ctx, a.op, lhs, a.imm, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Icmp(c) => { - let lhs = val_map[c.lhs.index()].expect("operand"); - let rhs = val_map[c.rhs.index()].expect("operand"); - let op = IcmpOp::new(&mut ctx, c.op, lhs, rhs, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Select(s) => { - let cond = val_map[s.cond.index()].expect("operand"); - let t = val_map[s.then.index()].expect("operand"); - let e = val_map[s.else_.index()].expect("operand"); - let op = SelectOp::new(&mut ctx, cond, t, e, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::StructNew(s) => { - let fields: Vec<_> = legacy - .value_list(s.fields) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let op = StructNewOp::new(&mut ctx, fields, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Extract(e) => { - let agg = val_map[e.aggregate.index()].expect("operand"); - let op = ExtractOp::new(&mut ctx, agg, e.index, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Convert(c) => { - let val = val_map[c.value.index()].expect("operand"); - let op = ConvertOp::new(&mut ctx, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::DecodeLowFloat(d) => { - let val = val_map[d.value.index()].expect("operand"); - let op = DecodeLowFloatOp::new(&mut ctx, d.format, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::EncodeLowFloat(e) => { - let val = val_map[e.value.index()].expect("operand"); - let op = EncodeLowFloatOp::new(&mut ctx, e.format, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::DequantizeNvfp4(q) => { - let val = val_map[q.value.index()].expect("operand"); - let b_scale = val_map[q.block_scale.index()].expect("operand"); - let g_scale = val_map[q.global_scale.index()].expect("operand"); - let op = DequantizeNvfp4Op::new( - &mut ctx, - LowFloatFormat::F8E4M3, - val, - b_scale, - g_scale, - q.block_application, - q.global_application, - pliron_ty, - ); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::QuantizeNvfp4(q) => { - let val = val_map[q.value.index()].expect("operand"); - let b_scale = val_map[q.block_scale.index()].expect("operand"); - let g_scale = val_map[q.global_scale.index()].expect("operand"); - let op = QuantizeNvfp4Op::new( - &mut ctx, - LowFloatFormat::F8E4M3, - val, - b_scale, - g_scale, - q.block_application, - q.global_application, - pliron_ty, - ); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Unary(u) => { - let val = val_map[u.value.index()].expect("operand"); - let op = UnaryOp::new(&mut ctx, u.op, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Alloca(a) => { - let elem_ty = type_map[a.elem.0 as usize]; - let op = AllocaOp::new(&mut ctx, elem_ty, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Call(c) => { - let callee = legacy.symbol_name(c.symbol); - let args: Vec<_> = legacy - .value_list(c.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let res_tys = legacy - .inst_result(*inst) - .map(|_| alloc::vec![pliron_ty]) - .unwrap_or_default(); - let call_info = CallInfo { - is_variadic: c.is_variadic, - num_fixed: c.num_fixed, - sret: c.sret, - ret_regs: c.ret_regs, - ret_pieces: c.ret_pieces.to_vec(), - }; - let op = CallOp::new(&mut ctx, callee, args, res_tys, call_info); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::CallIndirect(c) => { - let target = val_map[c.target.index()].expect("operand"); - let args: Vec<_> = legacy - .value_list(c.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let res_tys = legacy - .inst_result(*inst) - .map(|_| alloc::vec![pliron_ty]) - .unwrap_or_default(); - let call_info = CallInfo { - is_variadic: c.is_variadic, - num_fixed: c.num_fixed, - sret: c.sret, - ret_regs: c.ret_regs, - ret_pieces: c.ret_pieces.to_vec(), - }; - let op = CallIndirectOp::new(&mut ctx, target, args, res_tys, call_info); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::GlobalAddr(g) => { - let symbol = legacy.symbol_name(g.symbol); - let op = GlobalAddrOp::new(&mut ctx, symbol, g.via_got, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Load(l) => { - let ptr = val_map[l.ptr.index()].expect("operand"); - let op = LoadOp::new(&mut ctx, ptr, l.volatile, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Store(s) => { - let ptr = val_map[s.ptr.index()].expect("operand"); - let val = val_map[s.value.index()].expect("operand"); - let op = StoreOp::new(&mut ctx, val, ptr, s.volatile); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::Prefetch(p) => { - let ptr = val_map[p.ptr.index()].expect("operand"); - let op = PrefetchOp::new(&mut ctx, ptr); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::VaStart(v) => { - let list = val_map[v.list.index()].expect("operand"); - let op = VaStartOp::new(&mut ctx, list); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::VaArg(v) => { - let list = val_map[v.list.index()].expect("operand"); - let op = VaArgOp::new(&mut ctx, list, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::VaEnd(v) => { - let list = val_map[v.list.index()].expect("operand"); - let op = VaEndOp::new(&mut ctx, list); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::Dot(d) => { - let acc = val_map[d.acc.index()].expect("operand"); - let a = val_map[d.a.index()].expect("operand"); - let b = val_map[d.b.index()].expect("operand"); - let op = DotOp::new(&mut ctx, acc, a, b, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Reduce(r) => { - let v = val_map[r.vector.index()].expect("operand"); - let op = ReduceOp::new(&mut ctx, r.op, v, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Splat(s) => { - let v = val_map[s.scalar.index()].expect("operand"); - let op = SplatOp::new(&mut ctx, v, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Matmul(m) => { - let a = val_map[m.a.index()].expect("operand"); - let b = val_map[m.b.index()].expect("operand"); - let c = val_map[m.c.index()].expect("operand"); - let desc = convert_matmul_desc_to_pliron(legacy, &m); - let op = MatMulOp::new(&mut ctx, a, b, c, desc, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Barrier(b_op) => { - let op = BarrierOp::new(&mut ctx, b_op.scope); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::AtomicRmw(a) => { - let ptr = val_map[a.ptr.index()].expect("operand"); - let val = val_map[a.value.index()].expect("operand"); - let cmp = a.compare.map(|c| val_map[c.index()].expect("operand")); - let op = AtomicRmwOp::new( - &mut ctx, a.op, ptr, val, cmp, a.ordering, a.scope, pliron_ty, - ); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::If(cf) => { - let cond = val_map[cf.cond.index()].expect("operand"); - let then_dest = blocks[cf.then.target.index()]; - let then_args: Vec<_> = legacy - .value_list(cf.then.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let else_dest = blocks[cf.else_.target.index()]; - let else_args: Vec<_> = legacy - .value_list(cf.else_.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let op = CondBrOp::new( - &mut ctx, cond, then_dest, then_args, else_dest, else_args, - ); - op.get_operation().insert_at_back(bb, &mut ctx); - } - } - } - - // Emit terminator if present. - if let Some(term) = legacy.terminator(legacy_b) { - match term { - crate::function::Terminator::Ret(r) => { - let vals: Vec<_> = r - .slice() - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let op = RetOp::new(&mut ctx, vals); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Terminator::Jump(j) => { - let dest = blocks[j.target.index()]; - let args: Vec<_> = legacy - .value_list(j.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let op = JumpOp::new(&mut ctx, dest, args); - op.get_operation().insert_at_back(bb, &mut ctx); - } - } - } - } - - let mut res = Self { ctx, func }; - res.set_func_info(FuncInfo { - is_variadic: legacy.is_variadic, - num_fixed_params: legacy.num_fixed_params, - sret: legacy.sret, - is_local: legacy.is_local, - }); - - res - } - - /// Convert this pliron [`Function`] into a legacy [`crate::function::Function`]. - pub fn to_legacy(&self) -> Result { - let mut legacy = crate::function::Function::new(); - - if let Some(info) = self.func_info() { - legacy.is_variadic = info.is_variadic; - legacy.num_fixed_params = info.num_fixed_params; - legacy.sret = info.sret; - legacy.is_local = info.is_local; - } - - let region = self.func.get_region(&self.ctx); - let blocks: Vec<_> = region.deref(&self.ctx).iter(&self.ctx).collect(); - if blocks.is_empty() { - return Ok(legacy); - } - - // Map pliron basic blocks to legacy Block indices. - let mut block_map: HashMap, crate::function::Block> = HashMap::new(); - let mut val_map: HashMap = HashMap::new(); - - for (i, &bb) in blocks.iter().enumerate() { - let legacy_b = legacy.append_block(); - assert_eq!(legacy_b.index(), i); - block_map.insert(bb, legacy_b); - } - - // Create block parameters. - for &bb in &blocks { - let legacy_b = block_map[&bb]; - let bb_ref = bb.deref(&self.ctx); - for arg_idx in 0..bb_ref.get_num_arguments() { - let arg = bb_ref.get_argument(arg_idx); - let ty = arg.get_type(&self.ctx); - let legacy_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let legacy_v = legacy.append_block_param(legacy_b, legacy_ty); - val_map.insert(arg, legacy_v); - } - } - - // Walk instructions in each block. - for &bb in &blocks { - let legacy_b = block_map[&bb]; - let bb_ref = bb.deref(&self.ctx); - - for op_ptr in bb_ref.iter(&self.ctx) { - let op = op_ptr.deref(&self.ctx); - let op_id = Operation::get_opid(op_ptr, &self.ctx); - let op_name = op_id.to_string(); - - match op_name.as_str() { - "volt.iconst" => { - let c = ConstIntOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Iconst(c.value(&self.ctx)), - ); - val_map.insert(op.get_result(0), v); - } - "volt.fconst" => { - let c = ConstFloatOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Fconst(c.value(&self.ctx)), - ); - val_map.insert(op.get_result(0), v); - } - "volt.fconst128" => { - let c = ConstF128Op::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Fconst128(c.value(&self.ctx)), - ); - val_map.insert(op.get_result(0), v); - } - "volt.arith" => { - let a = ArithOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let lhs = *val_map - .get(&a.lhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let rhs = *val_map - .get(&a.rhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Arith(crate::function::Arith { - op: a.kind(&self.ctx), - lhs, - rhs, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.arith_imm" => { - let a = ArithImmOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let lhs = *val_map - .get(&a.lhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::ArithImm(crate::function::ArithImm { - op: a.kind(&self.ctx), - lhs, - imm: a.imm(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.icmp" => { - let c = IcmpOp::from_operation(op_ptr); - let bool_ty = legacy.types.intern(crate::types::TypeKind::Bool); - let lhs = *val_map - .get(&c.lhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let rhs = *val_map - .get(&c.rhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - bool_ty, - crate::function::Opcode::Icmp(crate::function::Compare { - op: c.kind(&self.ctx), - lhs, - rhs, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.select" => { - let s = SelectOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (cond_v, then_raw, else_raw) = s.operands3(&self.ctx); - let cond = *val_map.get(&cond_v).ok_or(ConvertError::MissingValue)?; - let then_v = *val_map.get(&then_raw).ok_or(ConvertError::MissingValue)?; - let else_v = *val_map.get(&else_raw).ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Select(crate::function::Select { - cond, - then: then_v, - else_: else_v, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.struct_new" => { - let sn = StructNewOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let fields: Result, _> = sn - .fields(&self.ctx) - .iter() - .map(|f| val_map.get(f).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let v = legacy.append_struct_new(legacy_b, l_ty, &fields?); - val_map.insert(op.get_result(0), v); - } - "volt.extract" => { - let e = ExtractOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let agg = *val_map - .get(&e.aggregate(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Extract(crate::function::Extract { - aggregate: agg, - index: e.index(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.convert" => { - let c = ConvertOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let val = *val_map - .get(&c.value(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Convert(crate::function::Convert { - value: val, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.decode_low_float" => { - let d = DecodeLowFloatOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let val = *val_map - .get(&d.value(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::DecodeLowFloat( - crate::function::LowFloatConvert { - format: d.float_format(&self.ctx), - value: val, - }, - ), - ); - val_map.insert(op.get_result(0), v); - } - "volt.encode_low_float" => { - let e = EncodeLowFloatOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let val = *val_map - .get(&e.value(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::EncodeLowFloat( - crate::function::LowFloatConvert { - format: e.float_format(&self.ctx), - value: val, - }, - ), - ); - val_map.insert(op.get_result(0), v); - } - "volt.dequantize_nvfp4" => { - let q = DequantizeNvfp4Op::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (_fmt, b_app, g_app, v_raw, bs_raw, gs_raw) = q.parts(&self.ctx); - let val = *val_map.get(&v_raw).ok_or(ConvertError::MissingValue)?; - let b_scale = *val_map.get(&bs_raw).ok_or(ConvertError::MissingValue)?; - let g_scale = *val_map.get(&gs_raw).ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::DequantizeNvfp4( - crate::function::NvFp4Convert { - block_application: b_app, - global_application: g_app, - value: val, - block_scale: b_scale, - global_scale: g_scale, - }, - ), - ); - val_map.insert(op.get_result(0), v); - } - "volt.quantize_nvfp4" => { - let q = QuantizeNvfp4Op::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (_fmt, b_app, g_app, v_raw, bs_raw, gs_raw) = q.parts(&self.ctx); - let val = *val_map.get(&v_raw).ok_or(ConvertError::MissingValue)?; - let b_scale = *val_map.get(&bs_raw).ok_or(ConvertError::MissingValue)?; - let g_scale = *val_map.get(&gs_raw).ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::QuantizeNvfp4(crate::function::NvFp4Convert { - block_application: b_app, - global_application: g_app, - value: val, - block_scale: b_scale, - global_scale: g_scale, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.unary" => { - let u = UnaryOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let val = *val_map - .get(&u.value(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Unary(crate::function::Unary { - op: u.kind(&self.ctx), - value: val, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.alloca" => { - let a = AllocaOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let elem = convert_pliron_type_to_legacy( - &self.ctx, - &mut legacy.types, - a.elem_type(&self.ctx), - )?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Alloca(crate::function::Alloca { elem }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.call" => { - let c = CallOp::from_operation(op_ptr); - let callee = c.callee(&self.ctx); - let info = c.info(&self.ctx); - let args: Result, _> = c - .args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let args_list = legacy.intern_values(&args?); - let symbol = legacy.intern_symbol(&callee); - let mut ret_pieces = [RetPiece::default(); 4]; - for (dest, src) in ret_pieces.iter_mut().zip(&info.ret_pieces) { - *dest = *src; - } - let call = crate::function::Call { - symbol, - args: args_list, - is_variadic: info.is_variadic, - num_fixed: info.num_fixed, - ret_dest: None, - ret_regs: info.ret_regs, - ret_pieces, - sret: info.sret, - }; - if op.get_num_results() > 0 { - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = - convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Call(call), - ); - val_map.insert(op.get_result(0), v); - } else { - legacy.append_stmt_raw(legacy_b, crate::function::Opcode::Call(call)); - } - } - "volt.call_indirect" => { - let c = CallIndirectOp::from_operation(op_ptr); - let target = *val_map - .get(&c.target(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let info = c.info(&self.ctx); - let args: Result, _> = c - .args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let args_list = legacy.intern_values(&args?); - let mut ret_pieces = [RetPiece::default(); 4]; - for (dest, src) in ret_pieces.iter_mut().zip(&info.ret_pieces) { - *dest = *src; - } - let call = crate::function::CallIndirect { - target, - args: args_list, - is_variadic: info.is_variadic, - num_fixed: info.num_fixed, - ret_dest: None, - ret_regs: info.ret_regs, - ret_pieces, - sret: info.sret, - }; - if op.get_num_results() > 0 { - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = - convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::CallIndirect(call), - ); - val_map.insert(op.get_result(0), v); - } else { - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::CallIndirect(call), - ); - } - } - "volt.global_addr" => { - let g = GlobalAddrOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let symbol = legacy.intern_symbol(&g.symbol(&self.ctx)); - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::GlobalAddr(crate::function::GlobalAddr { - symbol, - via_got: g.via_got(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.load" => { - let l = LoadOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let ptr = *val_map - .get(&l.ptr(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Load(crate::function::Load { - ptr, - volatile: l.volatile(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.store" => { - let s = StoreOp::from_operation(op_ptr); - let (v_raw, p_raw) = s.parts(&self.ctx); - let val = *val_map.get(&v_raw).ok_or(ConvertError::MissingValue)?; - let ptr = *val_map.get(&p_raw).ok_or(ConvertError::MissingValue)?; - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::Store(crate::function::Store { - ptr, - value: val, - volatile: s.volatile(&self.ctx), - }), - ); - } - "volt.prefetch" => { - let p = PrefetchOp::from_operation(op_ptr); - let ptr = *val_map - .get(&p.ptr(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::Prefetch(crate::function::Prefetch { ptr }), - ); - } - "volt.va_start" => { - let v = VaStartOp::from_operation(op_ptr); - let list = *val_map - .get(&v.list(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::VaStart(crate::function::VaStart { list }), - ); - } - "volt.va_arg" => { - let v = VaArgOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let list = *val_map - .get(&v.list(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let val = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::VaArg(crate::function::VaArg { - list, - ty: l_ty, - }), - ); - val_map.insert(op.get_result(0), val); - } - "volt.va_end" => { - let v = VaEndOp::from_operation(op_ptr); - let list = *val_map - .get(&v.list(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::VaEnd(crate::function::VaEnd { list }), - ); - } - "volt.dot" => { - let d = DotOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (acc_raw, a_raw, b_raw) = d.operands3(&self.ctx); - let acc = *val_map.get(&acc_raw).ok_or(ConvertError::MissingValue)?; - let a = *val_map.get(&a_raw).ok_or(ConvertError::MissingValue)?; - let b = *val_map.get(&b_raw).ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Dot(crate::function::Dot { acc, a, b }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.reduce" => { - let r = ReduceOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let vector = *val_map - .get(&r.vector(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Reduce(crate::function::Reduce { - op: r.kind(&self.ctx), - vector, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.splat" => { - let s = SplatOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let scalar = *val_map - .get(&s.scalar(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Splat(crate::function::Splat { scalar }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.matmul" => { - let m = MatMulOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (a_raw, b_raw, c_raw) = m.operands3(&self.ctx); - let a = *val_map.get(&a_raw).ok_or(ConvertError::MissingValue)?; - let b = *val_map.get(&b_raw).ok_or(ConvertError::MissingValue)?; - let c = *val_map.get(&c_raw).ok_or(ConvertError::MissingValue)?; - let desc = m.config(&self.ctx); - let mm = convert_matmul_desc_to_legacy(&mut legacy, &desc, a, b, c); - let v = - legacy.append_inst(legacy_b, l_ty, crate::function::Opcode::Matmul(mm)); - val_map.insert(op.get_result(0), v); - } - "volt.barrier" => { - let b_op = BarrierOp::from_operation(op_ptr); - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::Barrier(crate::function::Barrier { - scope: b_op.scope(&self.ctx), - }), - ); - } - "volt.atomic_rmw" => { - let a = AtomicRmwOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (p_raw, v_raw, cmp_raw) = a.operands_rmw(&self.ctx); - let ptr = *val_map.get(&p_raw).ok_or(ConvertError::MissingValue)?; - let value = *val_map.get(&v_raw).ok_or(ConvertError::MissingValue)?; - let compare = cmp_raw - .map(|c| val_map.get(&c).copied().ok_or(ConvertError::MissingValue)) - .transpose()?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::AtomicRmw(crate::function::AtomicRmw { - op: a.kind(&self.ctx), - ptr, - value, - compare, - ordering: a.ordering(&self.ctx), - scope: a.scope(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.cond_br" => { - let br = CondBrOp::from_operation(op_ptr); - let cond = *val_map - .get(&br.cond(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let then_target = *block_map - .get(&br.then_dest(&self.ctx)) - .ok_or(ConvertError::MissingBlock)?; - let then_args: Result, _> = br - .then_args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let else_target = *block_map - .get(&br.else_dest(&self.ctx)) - .ok_or(ConvertError::MissingBlock)?; - let else_args: Result, _> = br - .else_args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let then_edge = crate::function::EdgeDesc { - target: then_target, - args: then_args?, - }; - let else_edge = crate::function::EdgeDesc { - target: else_target, - args: else_args?, - }; - legacy.append_if(legacy_b, cond, then_edge, else_edge); - } - "volt.jump" => { - let j = JumpOp::from_operation(op_ptr); - let target = *block_map - .get(&j.target(&self.ctx)) - .ok_or(ConvertError::MissingBlock)?; - let args: Vec<_> = j - .args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect::, _>>()?; - let interned = legacy.intern_values(&args); - legacy.set_terminator( - legacy_b, - crate::function::Terminator::Jump(crate::function::Jump { - target, - args: interned, - }), - ); - } - "volt.ret" => { - let r = RetOp::from_operation(op_ptr); - let vals: Result, _> = r - .values(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let vals = vals?; - legacy.set_terminator( - legacy_b, - crate::function::Terminator::Ret(crate::function::Ret::many(&vals)), - ); - } - _ => return Err(ConvertError::UnsupportedOp(op_name)), - } - } - } - - Ok(legacy) - } -} - -/// An instruction builder for pliron functions. -pub struct Builder<'a> { - pub ctx: &'a mut Context, - pub block: Ptr, -} - -impl<'a> Builder<'a> { - /// Create a new builder in the given block. - pub fn new(ctx: &'a mut Context, block: Ptr) -> Self { - Self { ctx, block } - } - - /// Reposition the builder at the end of another block. - pub fn switch_to(&mut self, block: Ptr) { - self.block = block; - } - - /// The current block being appended to. - pub fn block(&self) -> Ptr { - self.block - } - - /// Emit an integer constant. - pub fn iconst(&mut self, val: i64, ty: TypeHandle) -> Value { - let op = ConstIntOp::new(self.ctx, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a double-precision float constant. - pub fn fconst(&mut self, val: f64, ty: TypeHandle) -> Value { - let op = ConstFloatOp::new(self.ctx, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit an f128 constant. - pub fn fconst128(&mut self, bits: u128, ty: TypeHandle) -> Value { - let op = ConstF128Op::new(self.ctx, bits, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit binary arithmetic. - pub fn arith(&mut self, kind: BinOp, lhs: Value, rhs: Value, ty: TypeHandle) -> Value { - let op = ArithOp::new(self.ctx, kind, lhs, rhs, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit binary arithmetic with an immediate. - pub fn arith_imm(&mut self, kind: BinOp, lhs: Value, imm: i64, ty: TypeHandle) -> Value { - let op = ArithImmOp::new(self.ctx, kind, lhs, imm, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit an integer comparison. - pub fn icmp(&mut self, kind: CmpOp, lhs: Value, rhs: Value) -> Value { - let bool_ty = BoolType::get_type(self.ctx); - let op = IcmpOp::new(self.ctx, kind, lhs, rhs, bool_ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a select. - pub fn select( - &mut self, - cond: Value, - then_val: Value, - else_val: Value, - ty: TypeHandle, - ) -> Value { - let op = SelectOp::new(self.ctx, cond, then_val, else_val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit struct construction. - pub fn struct_new(&mut self, fields: Vec, ty: TypeHandle) -> Value { - let op = StructNewOp::new(self.ctx, fields, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit struct field extraction. - pub fn extract(&mut self, aggregate: Value, index: u32, ty: TypeHandle) -> Value { - let op = ExtractOp::new(self.ctx, aggregate, index, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit type conversion. - pub fn convert(&mut self, val: Value, ty: TypeHandle) -> Value { - let op = ConvertOp::new(self.ctx, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit low-float decode. - pub fn decode_low_float(&mut self, fmt: LowFloatFormat, val: Value, ty: TypeHandle) -> Value { - let op = DecodeLowFloatOp::new(self.ctx, fmt, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit low-float encode. - pub fn encode_low_float(&mut self, fmt: LowFloatFormat, val: Value, ty: TypeHandle) -> Value { - let op = EncodeLowFloatOp::new(self.ctx, fmt, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit NVFP4 dequantize. - pub fn dequantize_nvfp4( - &mut self, - fmt: LowFloatFormat, - val: Value, - b_scale: Value, - g_scale: Value, - b_app: ScaleApplication, - g_app: ScaleApplication, - ty: TypeHandle, - ) -> Value { - let op = DequantizeNvfp4Op::new(self.ctx, fmt, val, b_scale, g_scale, b_app, g_app, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit NVFP4 quantize. - pub fn quantize_nvfp4( - &mut self, - fmt: LowFloatFormat, - val: Value, - b_scale: Value, - g_scale: Value, - b_app: ScaleApplication, - g_app: ScaleApplication, - ty: TypeHandle, - ) -> Value { - let op = QuantizeNvfp4Op::new(self.ctx, fmt, val, b_scale, g_scale, b_app, g_app, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a unary operation. - pub fn unary(&mut self, kind: crate::function::UnaryOp, val: Value, ty: TypeHandle) -> Value { - let op = UnaryOp::new(self.ctx, kind, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit alloca. - pub fn alloca(&mut self, elem_ty: TypeHandle, ptr_ty: TypeHandle) -> Value { - let op = AllocaOp::new(self.ctx, elem_ty, ptr_ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a direct call. - pub fn call( - &mut self, - callee: &str, - args: Vec, - res_tys: Vec, - info: CallInfo, - ) -> Vec { - let op = CallOp::new(self.ctx, callee, args, res_tys, info); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).results().collect() - } - - /// Emit an indirect call. - pub fn call_indirect( - &mut self, - target: Value, - args: Vec, - res_tys: Vec, - info: CallInfo, - ) -> Vec { - let op = CallIndirectOp::new(self.ctx, target, args, res_tys, info); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).results().collect() - } - - /// Emit a global address reference. - pub fn global_addr(&mut self, symbol: &str, via_got: bool, ty: TypeHandle) -> Value { - let op = GlobalAddrOp::new(self.ctx, symbol, via_got, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a load. - pub fn load(&mut self, ptr: Value, volatile: bool, ty: TypeHandle) -> Value { - let op = LoadOp::new(self.ctx, ptr, volatile, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a store. - pub fn store(&mut self, ptr: Value, val: Value, volatile: bool) { - let op = StoreOp::new(self.ctx, val, ptr, volatile); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit prefetch. - pub fn prefetch(&mut self, ptr: Value) { - let op = PrefetchOp::new(self.ctx, ptr); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit va_start. - pub fn va_start(&mut self, list: Value) { - let op = VaStartOp::new(self.ctx, list); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit va_arg. - pub fn va_arg(&mut self, list: Value, ty: TypeHandle) -> Value { - let op = VaArgOp::new(self.ctx, list, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit va_end. - pub fn va_end(&mut self, list: Value) { - let op = VaEndOp::new(self.ctx, list); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit dot product. - pub fn dot(&mut self, acc: Value, a: Value, b: Value, ty: TypeHandle) -> Value { - let op = DotOp::new(self.ctx, acc, a, b, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit vector reduction. - pub fn reduce(&mut self, kind: BinOp, vec: Value, ty: TypeHandle) -> Value { - let op = ReduceOp::new(self.ctx, kind, vec, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit vector splat. - pub fn splat(&mut self, val: Value, ty: TypeHandle) -> Value { - let op = SplatOp::new(self.ctx, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit matrix multiplication. - pub fn matmul( - &mut self, - a: Value, - b: Value, - c: Value, - desc: MatMulDesc, - ty: TypeHandle, - ) -> Value { - let op = MatMulOp::new(self.ctx, a, b, c, desc, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a barrier. - pub fn barrier(&mut self, scope: BarrierScope) { - let op = BarrierOp::new(self.ctx, scope); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit an atomic RMW operation. - pub fn atomic_rmw( - &mut self, - op: AtomicOp, - ptr: Value, - val: Value, - cmp: Option, - ordering: AtomicOrdering, - scope: AtomicScope, - ty: TypeHandle, - ) -> Value { - let op = AtomicRmwOp::new(self.ctx, op, ptr, val, cmp, ordering, scope, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit an unconditional jump. - pub fn jump(&mut self, target: Ptr, args: Vec) { - let op = JumpOp::new(self.ctx, target, args); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit a conditional branch. - pub fn cond_br( - &mut self, - cond: Value, - then_dest: Ptr, - then_args: Vec, - else_dest: Ptr, - else_args: Vec, - ) { - let op = CondBrOp::new(self.ctx, cond, then_dest, then_args, else_dest, else_args); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit a return. - pub fn ret(&mut self, values: Vec) { - let op = RetOp::new(self.ctx, values); - op.get_operation().insert_at_back(self.block, self.ctx); - } -} - -fn convert_legacy_type_to_pliron( - ctx: &mut Context, - types: &crate::types::TypeTable, - ty: crate::types::Type, -) -> TypeHandle { - match types.type_kind(ty) { - crate::types::TypeKind::Bool => BoolType::get_type(ctx), - crate::types::TypeKind::Int(desc) => IntType::get_type(ctx, desc.signed, desc.bits as u32), - crate::types::TypeKind::Float(k) => FloatType::get_type(ctx, *k), - crate::types::TypeKind::Ptr(s) => PtrType::get_type(ctx, *s), - crate::types::TypeKind::Vector(v) => { - let elem = convert_legacy_type_to_pliron(ctx, types, v.elem); - VectorType::get_type(ctx, v.len, elem) - } - crate::types::TypeKind::Array(a) => { - let elem = convert_legacy_type_to_pliron(ctx, types, a.elem); - ArrayType::get_type(ctx, a.len, elem) - } - crate::types::TypeKind::Slice(s) => { - let elem = convert_legacy_type_to_pliron(ctx, types, s.elem); - SliceType::get_type(ctx, elem) - } - crate::types::TypeKind::Struct(fields) => { - let field_tys: Vec<_> = fields - .iter() - .map(|&f| convert_legacy_type_to_pliron(ctx, types, f)) - .collect(); - StructType::get_type(ctx, field_tys) - } - } -} - -fn convert_pliron_type_to_legacy( - ctx: &Context, - types: &mut crate::types::TypeTable, - ty: TypeHandle, -) -> Result { - let kind = describe(ctx, ty).ok_or_else(|| { - ConvertError::UnsupportedType(alloc::format!( - "{}", - ty.deref(ctx).print(ctx, &Default::default()) - )) - })?; - - let l_kind = match kind { - VoltKind::Bool => crate::types::TypeKind::Bool, - VoltKind::Int { signed, bits } => crate::types::TypeKind::Int(crate::types::IntDesc { - signed, - bits: bits as u16, - }), - VoltKind::Float(k) => crate::types::TypeKind::Float(k), - VoltKind::Ptr(s) => crate::types::TypeKind::Ptr(s), - VoltKind::Vector { len, elem } => { - let e = convert_pliron_type_to_legacy(ctx, types, elem)?; - crate::types::TypeKind::Vector(crate::types::VectorDesc { len, elem: e }) - } - VoltKind::Array { len, elem } => { - let e = convert_pliron_type_to_legacy(ctx, types, elem)?; - crate::types::TypeKind::Array(crate::types::ArrayDesc { len, elem: e }) - } - VoltKind::Slice { elem } => { - let e = convert_pliron_type_to_legacy(ctx, types, elem)?; - crate::types::TypeKind::Slice(crate::types::SliceDesc { elem: e }) - } - VoltKind::Struct(fields) => { - let mut l_fields = Vec::with_capacity(fields.len()); - for f in fields { - l_fields.push(convert_pliron_type_to_legacy(ctx, types, f)?); - } - crate::types::TypeKind::Struct(l_fields) - } - }; - Ok(types.intern(l_kind)) -} - -fn convert_matmul_desc_to_pliron( - legacy: &crate::function::Function, - m: &crate::function::MatMul, -) -> MatMulDesc { - MatMulDesc { - m: m.m, - n: m.n, - k: m.k, - dtype: m.dtype, - accumulate: m.accumulate, - embedded: m.embedded, - quant: m.quant.as_ref().map(|q| { - let (scalar, col) = match q.scale { - crate::function::MatMulScale::Scalar(s) => (Some(s), None), - crate::function::MatMulScale::PerColumn(cl) => { - (None, Some(legacy.scale_list(cl).to_vec())) - } - }; - QuantDesc { - scale_scalar: scalar, - scale_per_column: col, - relu: q.relu, - out: q.out, - bias: q.bias.map(|b| legacy.bias_list(b).to_vec()), - zero_point: q.zero_point, - } - }), - input_signs: m.input_signs, - } -} - -fn convert_matmul_desc_to_legacy( - legacy: &mut crate::function::Function, - desc: &MatMulDesc, - a: crate::function::Value, - b: crate::function::Value, - c: crate::function::Value, -) -> crate::function::MatMul { - let quant = desc.quant.as_ref().map(|q| { - let scale = if let Some(s) = q.scale_scalar { - crate::function::MatMulScale::Scalar(s) - } else if let Some(cols) = &q.scale_per_column { - crate::function::MatMulScale::PerColumn(legacy.intern_scales(cols)) - } else { - crate::function::MatMulScale::Scalar(0) - }; - let bias = q.bias.as_ref().map(|biases| legacy.intern_bias(biases)); - crate::function::MatMulQuant { - scale, - relu: q.relu, - out: q.out, - bias, - zero_point: q.zero_point, - } - }); - - crate::function::MatMul { - a, - b, - c, - m: desc.m, - n: desc.n, - k: desc.k, - dtype: desc.dtype, - accumulate: desc.accumulate, - embedded: desc.embedded, - quant, - input_signs: desc.input_signs, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn builder_emits_valid_function() { - let mut f = Function::new("test_fn", vec![], vec![]); - let i32t = IntType::get_type(&f.ctx, true, 32); - let entry = f.entry_block(); - let mut b = f.builder(entry); - let v1 = b.iconst(10, i32t); - let v2 = b.iconst(20, i32t); - let sum = b.arith(BinOp::Add, v1, v2, i32t); - b.ret(vec![sum]); - - let text = f.print(); - assert!(text.contains("volt.iconst 10"), "{text}"); - assert!(text.contains("volt.iconst 20"), "{text}"); - assert!(text.contains("volt.arith add"), "{text}"); - assert!(text.contains("volt.ret"), "{text}"); - assert!(f.verify().is_ok()); - } - - #[test] - fn round_trip_legacy_function() { - let mut legacy = crate::function::Function::new(); - let i32_t = legacy - .types - .intern(crate::types::TypeKind::Int(crate::types::IntDesc { - signed: true, - bits: 32, - })); - let b0 = legacy.append_block(); - let p0 = legacy.append_block_param(b0, i32_t); - let c1 = legacy.append_inst(b0, i32_t, crate::function::Opcode::Iconst(42)); - let add = legacy.append_inst( - b0, - i32_t, - crate::function::Opcode::Arith(crate::function::Arith { - op: BinOp::Add, - lhs: p0, - rhs: c1, - }), - ); - legacy.set_terminator( - b0, - crate::function::Terminator::Ret(crate::function::Ret::one(add)), - ); - - let pliron_f = Function::from_legacy(&legacy); - assert!(pliron_f.verify().is_ok()); - - let round_tripped = pliron_f.to_legacy().unwrap(); - assert_eq!(round_tripped.block_count(), legacy.block_count()); - assert_eq!(round_tripped.block_params(b0).len(), 1); - assert_eq!(round_tripped.block_insts(b0).len(), 2); - } - - #[test] - fn round_trip_control_flow() { - let mut legacy = crate::function::Function::new(); - let bool_t = legacy.types.intern(crate::types::TypeKind::Bool); - let i32_t = legacy - .types - .intern(crate::types::TypeKind::Int(crate::types::IntDesc { - signed: true, - bits: 32, - })); - let b0 = legacy.append_block(); - let cond = legacy.append_block_param(b0, bool_t); - let b1 = legacy.append_block(); - let p1 = legacy.append_block_param(b1, i32_t); - let b2 = legacy.append_block(); - let p2 = legacy.append_block_param(b2, i32_t); - - let c1 = legacy.append_inst(b0, i32_t, crate::function::Opcode::Iconst(1)); - let c2 = legacy.append_inst(b0, i32_t, crate::function::Opcode::Iconst(2)); - - let then_edge = crate::function::EdgeDesc { - target: b1, - args: vec![c1], - }; - let else_edge = crate::function::EdgeDesc { - target: b2, - args: vec![c2], - }; - legacy.append_if(b0, cond, then_edge, else_edge); - legacy.set_terminator( - b1, - crate::function::Terminator::Ret(crate::function::Ret::one(p1)), - ); - legacy.set_terminator( - b2, - crate::function::Terminator::Ret(crate::function::Ret::one(p2)), - ); - - let pliron_f = Function::from_legacy(&legacy); - assert!(pliron_f.verify().is_ok()); - - let round_tripped = pliron_f.to_legacy().unwrap(); - assert_eq!(round_tripped.block_count(), 3); - assert_eq!(round_tripped.block_params(b1).len(), 1); - assert_eq!(round_tripped.block_params(b2).len(), 1); - } - - #[test] - fn round_trip_memory_ops() { - let mut legacy = crate::function::Function::new(); - let i32_t = legacy - .types - .intern(crate::types::TypeKind::Int(crate::types::IntDesc { - signed: true, - bits: 32, - })); - let ptr_t = legacy.types.intern(crate::types::TypeKind::Ptr( - crate::types::AddressSpace::Global, - )); - let b0 = legacy.append_block(); - let slot = legacy.append_inst( - b0, - ptr_t, - crate::function::Opcode::Alloca(crate::function::Alloca { elem: i32_t }), - ); - let c42 = legacy.append_inst(b0, i32_t, crate::function::Opcode::Iconst(42)); - legacy.append_store(b0, c42, slot); - let loaded = legacy.append_inst( - b0, - i32_t, - crate::function::Opcode::Load(crate::function::Load { - ptr: slot, - volatile: false, - }), - ); - legacy.set_terminator( - b0, - crate::function::Terminator::Ret(crate::function::Ret::one(loaded)), - ); - - let pliron_f = Function::from_legacy(&legacy); - assert!(pliron_f.verify().is_ok()); - - let round_tripped = pliron_f.to_legacy().unwrap(); - assert_eq!(round_tripped.block_count(), 1); - assert_eq!(round_tripped.block_insts(b0).len(), 4); - } -} diff --git a/src/ir/src/dialect/mod.rs b/src/ir/src/dialect/mod.rs deleted file mode 100644 index a4ab5c3..0000000 --- a/src/ir/src/dialect/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! The `volt` dialect: volt IR modeled with pliron. -//! -//! [`types`] defines the volt type system as pliron types, [`attrs`] -//! defines operation payload attributes, [`ops`] defines every volt -//! operation, and [`func`] provides the function-level container and -//! builder used by passes, targets, and the emulator. - -pub mod attrs; -pub mod func; -pub mod ops; -pub mod types; - -/// The volt dialect name, used as the `volt.` prefix on ops, types, -/// and attributes. -pub const DIALECT_NAME: &str = "volt"; diff --git a/src/ir/src/dialect/ops.rs b/src/ir/src/dialect/ops.rs deleted file mode 100644 index 381a707..0000000 --- a/src/ir/src/dialect/ops.rs +++ /dev/null @@ -1,1674 +0,0 @@ -//! Every `volt` dialect operation. -//! -//! The op set mirrors the legacy [`Opcode`](crate::function::Opcode): -//! constants, arithmetic, control flow, memory, calls, vector/matrix and -//! atomics. Most ops use a declarative assembly format; [`CondBrOp`] -//! (two successors with partitioned operands) has a manual -//! printer/parser. - -use alloc::string::String; -use alloc::vec::Vec; - -use pliron::{ - builtin::{ - attributes::{BoolAttr, StringAttr, TypeAttr}, - op_interfaces::{ - IsTerminatorInterface, NOpdsInterface, NResultsInterface, OneResultInterface, - }, - }, - combine::Parser, - combine::parser::token::token, - common_traits::Named, - context::{Context, Ptr}, - derive::pliron_op, - identifier::Identifier, - input_err, - irfmt::parsers::{ - block_opd_parser, delimited_list_parser, process_parsed_ssa_defs, spaced, ssa_opd_parser, - }, - location::{Located, Location}, - op::{Op, OpObj}, - operation::Operation, - parsable::{IntoParseResult, Parsable, ParseResult, StateStream}, - printable::{self, Printable}, - r#type::TypeHandle, - value::Value, -}; - -use super::super::function::{AtomicOp, AtomicOrdering, AtomicScope, BarrierScope, BinOp, CmpOp}; -use super::super::low_float::Format as LowFloatFormat; -use super::super::nvfp4::ScaleApplication; -use super::attrs::{ - AtomicOpAttr, AtomicOrderingAttr, AtomicScopeAttr, BarrierAttr, BinOpAttr, CallInfo, - CallInfoAttr, CmpOpAttr, F128BitsAttr, FloatBitsAttr, ImmAttr, LowFloatAttr, MatMulAttr, - MatMulDesc, ScaleAppAttr, UnaryOpAttr, -}; -use pliron::basic_block::BasicBlock; - -// Fetch a required attribute, panicking with the op name on absence. -// Attributes are always set by constructors; absence is a bug. -fn req_attr(opt: Option>, op: &str, attr: &str) -> T { - opt.map(|r| T::clone(&r)) - .unwrap_or_else(|| panic!("{op} is missing required attribute {attr}")) -} - -/// An integer constant: `%r = volt.iconst 42 : volt.int`. -#[pliron_op( - name = "volt.iconst", - format = "attr($iconst_value, $ImmAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (iconst_value: ImmAttr), - verifier = "succ", -)] -pub struct ConstIntOp; - -impl ConstIntOp { - /// Build an integer constant. - pub fn new(ctx: &mut Context, value: i64, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_iconst_value(ctx, ImmAttr::new(value)); - op - } - - /// The constant value. - pub fn value(&self, ctx: &Context) -> i64 { - req_attr( - self.get_attr_iconst_value(ctx), - "volt.iconst", - "iconst_value", - ) - .value() - } -} - -/// A double-precision constant (stored as raw bits). -#[pliron_op( - name = "volt.fconst", - format = "attr($fconst_value, $FloatBitsAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (fconst_value: FloatBitsAttr), - verifier = "succ", -)] -pub struct ConstFloatOp; - -impl ConstFloatOp { - /// Build a float constant. - pub fn new(ctx: &mut Context, value: f64, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_fconst_value(ctx, FloatBitsAttr::new(value)); - op - } - - /// The constant value. - pub fn value(&self, ctx: &Context) -> f64 { - req_attr( - self.get_attr_fconst_value(ctx), - "volt.fconst", - "fconst_value", - ) - .value() - } -} - -/// A quad-precision constant (stored as raw bits). -#[pliron_op( - name = "volt.fconst128", - format = "attr($fconst128_value, $F128BitsAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (fconst128_value: F128BitsAttr), - verifier = "succ", -)] -pub struct ConstF128Op; - -impl ConstF128Op { - /// Build an f128 constant. - pub fn new(ctx: &mut Context, value: u128, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_fconst128_value(ctx, F128BitsAttr::new(value)); - op - } - - /// The constant value. - pub fn value(&self, ctx: &Context) -> u128 { - req_attr( - self.get_attr_fconst128_value(ctx), - "volt.fconst128", - "fconst128_value", - ) - .value() - } -} - -/// Binary arithmetic: `volt.arith add %lhs, %rhs : T`. -#[pliron_op( - name = "volt.arith", - format = "attr($arith_kind, $BinOpAttr) ` ` $0 `, ` $1 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (arith_kind: BinOpAttr), - verifier = "succ", -)] -pub struct ArithOp; - -impl ArithOp { - /// Build a binary arithmetic op. - pub fn new(ctx: &mut Context, kind: BinOp, lhs: Value, rhs: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![lhs, rhs], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_arith_kind(ctx, BinOpAttr::new(kind)); - op - } - - /// The arithmetic operator. - pub fn kind(&self, ctx: &Context) -> BinOp { - req_attr(self.get_attr_arith_kind(ctx), "volt.arith", "arith_kind").value() - } - - /// Left operand. - pub fn lhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Right operand. - pub fn rhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(1) - } -} - -/// Arithmetic with an immediate: `volt.arith_imm add %lhs, 5 : T`. -#[pliron_op( - name = "volt.arith_imm", - format = "attr($arith_imm_kind, $BinOpAttr) ` ` $0 `, ` attr($arith_imm_imm, $ImmAttr) ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (arith_imm_kind: BinOpAttr, arith_imm_imm: ImmAttr), - verifier = "succ", -)] -pub struct ArithImmOp; - -impl ArithImmOp { - /// Build an arithmetic-immediate op. - pub fn new(ctx: &mut Context, kind: BinOp, lhs: Value, imm: i64, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![lhs], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_arith_imm_kind(ctx, BinOpAttr::new(kind)); - op.set_attr_arith_imm_imm(ctx, ImmAttr::new(imm)); - op - } - - /// The arithmetic operator. - pub fn kind(&self, ctx: &Context) -> BinOp { - req_attr( - self.get_attr_arith_imm_kind(ctx), - "volt.arith_imm", - "arith_imm_kind", - ) - .value() - } - - /// The value operand. - pub fn lhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// The immediate. - pub fn imm(&self, ctx: &Context) -> i64 { - req_attr( - self.get_attr_arith_imm_imm(ctx), - "volt.arith_imm", - "arith_imm_imm", - ) - .value() - } -} - -/// Integer comparison: `volt.icmp lt %lhs, %rhs : volt.bool`. -#[pliron_op( - name = "volt.icmp", - format = "attr($icmp_kind, $CmpOpAttr) ` ` $0 `, ` $1 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (icmp_kind: CmpOpAttr), - verifier = "succ", -)] -pub struct IcmpOp; - -impl IcmpOp { - /// Build a comparison. - pub fn new(ctx: &mut Context, kind: CmpOp, lhs: Value, rhs: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![lhs, rhs], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_icmp_kind(ctx, CmpOpAttr::new(kind)); - op - } - - /// The comparison operator. - pub fn kind(&self, ctx: &Context) -> CmpOp { - req_attr(self.get_attr_icmp_kind(ctx), "volt.icmp", "icmp_kind").value() - } - - /// Left operand. - pub fn lhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Right operand. - pub fn rhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(1) - } -} - -/// Conditional select: `volt.select %c, %then, %else : T`. -#[pliron_op( - name = "volt.select", - format = "$0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct SelectOp; - -impl SelectOp { - /// Build a select. - pub fn new(ctx: &mut Context, cond: Value, then: Value, else_: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![cond, then, else_], - Vec::new(), - 0, - ); - Self { op } - } - - /// Condition, then-value, else-value. - pub fn operands3(&self, ctx: &Context) -> (Value, Value, Value) { - let op = self.get_operation().deref(ctx); - (op.get_operand(0), op.get_operand(1), op.get_operand(2)) - } -} - -/// Struct construction from field values. -#[pliron_op( - name = "volt.struct_new", - format = "operands(CharSpace(`,`)) ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct StructNewOp; - -impl StructNewOp { - /// Build a struct value. - pub fn new(ctx: &mut Context, fields: Vec, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - fields, - Vec::new(), - 0, - ); - Self { op } - } - - /// Field values. - pub fn fields(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().collect() - } -} - -/// Struct field extraction: `volt.extract %agg 1 : T`. -#[pliron_op( - name = "volt.extract", - format = "$0 attr($extract_index, $ImmAttr) ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (extract_index: ImmAttr), - verifier = "succ", -)] -pub struct ExtractOp; - -impl ExtractOp { - /// Build a field extraction. - pub fn new(ctx: &mut Context, aggregate: Value, index: u32, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![aggregate], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_extract_index(ctx, ImmAttr::new(index as i64)); - op - } - - /// The aggregate value. - pub fn aggregate(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Field index. - pub fn index(&self, ctx: &Context) -> u32 { - req_attr( - self.get_attr_extract_index(ctx), - "volt.extract", - "extract_index", - ) - .value() as u32 - } -} - -/// Type conversion (int/float resize, int<->float, bitcast flavors). -#[pliron_op( - name = "volt.convert", - format = "$0 ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct ConvertOp; - -impl ConvertOp { - /// Build a conversion. - pub fn new(ctx: &mut Context, value: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value], - Vec::new(), - 0, - ); - Self { op } - } - - /// The converted value. - pub fn value(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Low-precision float decode: `volt.decode_low_float bf16 %v : T`. -#[pliron_op( - name = "volt.decode_low_float", - format = "attr($decode_format, $LowFloatAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (decode_format: LowFloatAttr), - verifier = "succ", -)] -pub struct DecodeLowFloatOp; - -impl DecodeLowFloatOp { - /// Build a decode. - pub fn new(ctx: &mut Context, format: LowFloatFormat, value: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_decode_format(ctx, LowFloatAttr::new(format)); - op - } - - /// The source format. - pub fn float_format(&self, ctx: &Context) -> LowFloatFormat { - req_attr( - self.get_attr_decode_format(ctx), - "volt.decode_low_float", - "decode_format", - ) - .value() - } - - /// The encoded value. - pub fn value(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Low-precision float encode. -#[pliron_op( - name = "volt.encode_low_float", - format = "attr($encode_format, $LowFloatAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (encode_format: LowFloatAttr), - verifier = "succ", -)] -pub struct EncodeLowFloatOp; - -impl EncodeLowFloatOp { - /// Build an encode. - pub fn new(ctx: &mut Context, format: LowFloatFormat, value: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_encode_format(ctx, LowFloatAttr::new(format)); - op - } - - /// The target format. - pub fn float_format(&self, ctx: &Context) -> LowFloatFormat { - req_attr( - self.get_attr_encode_format(ctx), - "volt.encode_low_float", - "encode_format", - ) - .value() - } - - /// The decoded value. - pub fn value(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// NVFP4 dequantize: value + block/global scales. -#[pliron_op( - name = "volt.dequantize_nvfp4", - format = "attr($dequant_format, $LowFloatAttr) ` ` attr($dequant_block_app, $ScaleAppAttr) ` ` attr($dequant_global_app, $ScaleAppAttr) ` ` $0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (dequant_format: LowFloatAttr, dequant_block_app: ScaleAppAttr, dequant_global_app: ScaleAppAttr), - verifier = "succ", -)] -pub struct DequantizeNvfp4Op; - -impl DequantizeNvfp4Op { - /// Build a dequantize. - #[allow(clippy::too_many_arguments)] - pub fn new( - ctx: &mut Context, - format: LowFloatFormat, - value: Value, - block_scale: Value, - global_scale: Value, - block_app: ScaleApplication, - global_app: ScaleApplication, - ty: TypeHandle, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value, block_scale, global_scale], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_dequant_format(ctx, LowFloatAttr::new(format)); - op.set_attr_dequant_block_app(ctx, ScaleAppAttr::new(block_app)); - op.set_attr_dequant_global_app(ctx, ScaleAppAttr::new(global_app)); - op - } - - /// Conversion parameters and operands (value, block scale, global scale). - pub fn parts( - &self, - ctx: &Context, - ) -> ( - LowFloatFormat, - ScaleApplication, - ScaleApplication, - Value, - Value, - Value, - ) { - let op = self.get_operation(); - let format = req_attr( - self.get_attr_dequant_format(ctx), - "volt.dequantize_nvfp4", - "dequant_format", - ) - .value(); - let block_app = req_attr( - self.get_attr_dequant_block_app(ctx), - "volt.dequantize_nvfp4", - "dequant_block_app", - ) - .value(); - let global_app = req_attr( - self.get_attr_dequant_global_app(ctx), - "volt.dequantize_nvfp4", - "dequant_global_app", - ) - .value(); - let op = op.deref(ctx); - ( - format, - block_app, - global_app, - op.get_operand(0), - op.get_operand(1), - op.get_operand(2), - ) - } -} - -/// NVFP4 quantize: value + block/global scales. -#[pliron_op( - name = "volt.quantize_nvfp4", - format = "attr($quant_format, $LowFloatAttr) ` ` attr($quant_block_app, $ScaleAppAttr) ` ` attr($quant_global_app, $ScaleAppAttr) ` ` $0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (quant_format: LowFloatAttr, quant_block_app: ScaleAppAttr, quant_global_app: ScaleAppAttr), - verifier = "succ", -)] -pub struct QuantizeNvfp4Op; - -impl QuantizeNvfp4Op { - /// Build a quantize. - #[allow(clippy::too_many_arguments)] - pub fn new( - ctx: &mut Context, - format: LowFloatFormat, - value: Value, - block_scale: Value, - global_scale: Value, - block_app: ScaleApplication, - global_app: ScaleApplication, - ty: TypeHandle, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value, block_scale, global_scale], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_quant_format(ctx, LowFloatAttr::new(format)); - op.set_attr_quant_block_app(ctx, ScaleAppAttr::new(block_app)); - op.set_attr_quant_global_app(ctx, ScaleAppAttr::new(global_app)); - op - } - - /// Conversion parameters and operands (value, block scale, global scale). - pub fn parts( - &self, - ctx: &Context, - ) -> ( - LowFloatFormat, - ScaleApplication, - ScaleApplication, - Value, - Value, - Value, - ) { - let op = self.get_operation(); - let format = req_attr( - self.get_attr_quant_format(ctx), - "volt.quantize_nvfp4", - "quant_format", - ) - .value(); - let block_app = req_attr( - self.get_attr_quant_block_app(ctx), - "volt.quantize_nvfp4", - "quant_block_app", - ) - .value(); - let global_app = req_attr( - self.get_attr_quant_global_app(ctx), - "volt.quantize_nvfp4", - "quant_global_app", - ) - .value(); - let op = op.deref(ctx); - ( - format, - block_app, - global_app, - op.get_operand(0), - op.get_operand(1), - op.get_operand(2), - ) - } -} - -/// Unary float/int op: `volt.unary sqrt %v : T`. -#[pliron_op( - name = "volt.unary", - format = "attr($unary_kind, $UnaryOpAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (unary_kind: UnaryOpAttr), - verifier = "succ", -)] -pub struct UnaryOp; - -impl UnaryOp { - /// Build a unary op. - pub fn new( - ctx: &mut Context, - kind: super::super::function::UnaryOp, - value: Value, - ty: TypeHandle, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_unary_kind(ctx, UnaryOpAttr::new(kind)); - op - } - - /// The unary operator. - pub fn kind(&self, ctx: &Context) -> super::super::function::UnaryOp { - req_attr(self.get_attr_unary_kind(ctx), "volt.unary", "unary_kind").value() - } - - /// The operand. - pub fn value(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Stack allocation of one element. -#[pliron_op( - name = "volt.alloca", - format = "attr($alloca_elem, $TypeAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (alloca_elem: TypeAttr), - verifier = "succ", -)] -pub struct AllocaOp; - -impl AllocaOp { - /// Build an alloca. - pub fn new(ctx: &mut Context, elem: TypeHandle, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_alloca_elem(ctx, TypeAttr::new(elem)); - op - } - - /// Element type. - pub fn elem_type(&self, ctx: &Context) -> TypeHandle { - TypeHandle::from(req_attr( - self.get_attr_alloca_elem(ctx), - "volt.alloca", - "alloca_elem", - )) - } -} - -/// Direct call: `volt.call @"sym"(%args) : T`. -/// Operand 0.. are the call arguments; there are 0 or 1 results. -#[pliron_op( - name = "volt.call", - format = "`@` attr($call_callee, $StringAttr) `(` operands(CharSpace(`,`)) `)` ` ` attr($call_info, $CallInfoAttr) ` : ` types(CharSpace(`,`))", - attributes = (call_callee: StringAttr, call_info: CallInfoAttr), - verifier = "succ", -)] -pub struct CallOp; - -impl CallOp { - /// Build a direct call. - pub fn new( - ctx: &mut Context, - callee: &str, - args: Vec, - results: Vec, - info: CallInfo, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - results, - args, - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_call_callee(ctx, StringAttr::new(String::from(callee))); - op.set_attr_call_info(ctx, CallInfoAttr::new(info)); - op - } - - /// Callee symbol name. - pub fn callee(&self, ctx: &Context) -> String { - req_attr(self.get_attr_call_callee(ctx), "volt.call", "call_callee") - .as_str() - .into() - } - - /// Call ABI details. - pub fn info(&self, ctx: &Context) -> CallInfo { - req_attr(self.get_attr_call_info(ctx), "volt.call", "call_info") - .info() - .clone() - } - - /// Call arguments. - pub fn args(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().collect() - } -} - -/// Indirect call through a function pointer. Operand 0 is the target. -#[pliron_op( - name = "volt.call_indirect", - format = "operands(CharSpace(`,`)) ` ` attr($call_indirect_info, $CallInfoAttr) ` : ` types(CharSpace(`,`))", - attributes = (call_indirect_info: CallInfoAttr), - verifier = "succ", -)] -pub struct CallIndirectOp; - -impl CallIndirectOp { - /// Build an indirect call. - pub fn new( - ctx: &mut Context, - target: Value, - args: Vec, - results: Vec, - info: CallInfo, - ) -> Self { - let mut operands = alloc::vec![target]; - operands.extend(args); - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - results, - operands, - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_call_indirect_info(ctx, CallInfoAttr::new(info)); - op - } - - /// Call target. - pub fn target(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Call ABI details. - pub fn info(&self, ctx: &Context) -> CallInfo { - req_attr( - self.get_attr_call_indirect_info(ctx), - "volt.call_indirect", - "call_indirect_info", - ) - .info() - .clone() - } - - /// Call arguments (excluding the target). - pub fn args(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().skip(1).collect() - } -} - -/// Address of a global symbol. -#[pliron_op( - name = "volt.global_addr", - format = "`@` attr($global_symbol, $StringAttr) ` ` attr($global_via_got, $BoolAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (global_symbol: StringAttr, global_via_got: BoolAttr), - verifier = "succ", -)] -pub struct GlobalAddrOp; - -impl GlobalAddrOp { - /// Build a global address. - pub fn new(ctx: &mut Context, symbol: &str, via_got: bool, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_global_symbol(ctx, StringAttr::new(String::from(symbol))); - op.set_attr_global_via_got(ctx, BoolAttr::new(via_got)); - op - } - - /// Symbol name. - pub fn symbol(&self, ctx: &Context) -> String { - req_attr( - self.get_attr_global_symbol(ctx), - "volt.global_addr", - "global_symbol", - ) - .as_str() - .into() - } - - /// Route through the GOT. - pub fn via_got(&self, ctx: &Context) -> bool { - req_attr( - self.get_attr_global_via_got(ctx), - "volt.global_addr", - "global_via_got", - ) - .into() - } -} - -/// Memory load: `volt.load false %ptr : T`. -#[pliron_op( - name = "volt.load", - format = "attr($load_volatile, $BoolAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (load_volatile: BoolAttr), - verifier = "succ", -)] -pub struct LoadOp; - -impl LoadOp { - /// Build a load. - pub fn new(ctx: &mut Context, ptr: Value, volatile: bool, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![ptr], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_load_volatile(ctx, BoolAttr::new(volatile)); - op - } - - /// Pointer operand. - pub fn ptr(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Volatile access. - pub fn volatile(&self, ctx: &Context) -> bool { - req_attr( - self.get_attr_load_volatile(ctx), - "volt.load", - "load_volatile", - ) - .into() - } -} - -/// Memory store: `volt.store false %value, %ptr`. -#[pliron_op( - name = "volt.store", - format = "attr($store_volatile, $BoolAttr) ` ` $0 `, ` $1", - interfaces = [NOpdsInterface<2>, NResultsInterface<0>], - attributes = (store_volatile: BoolAttr), - verifier = "succ", -)] -pub struct StoreOp; - -impl StoreOp { - /// Build a store. - pub fn new(ctx: &mut Context, value: Value, ptr: Value, volatile: bool) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - alloc::vec![value, ptr], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_store_volatile(ctx, BoolAttr::new(volatile)); - op - } - - /// Stored value and pointer. - pub fn parts(&self, ctx: &Context) -> (Value, Value) { - let op = self.get_operation().deref(ctx); - (op.get_operand(0), op.get_operand(1)) - } - - /// Volatile access. - pub fn volatile(&self, ctx: &Context) -> bool { - req_attr( - self.get_attr_store_volatile(ctx), - "volt.store", - "store_volatile", - ) - .into() - } -} - -/// Cache prefetch hint. -#[pliron_op( - name = "volt.prefetch", - format = "$0", - interfaces = [NOpdsInterface<1>, NResultsInterface<0>], - verifier = "succ", -)] -pub struct PrefetchOp; - -impl PrefetchOp { - /// Build a prefetch. - pub fn new(ctx: &mut Context, ptr: Value) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - alloc::vec![ptr], - Vec::new(), - 0, - ); - Self { op } - } - - /// Pointer operand. - pub fn ptr(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Start of variadic argument access. -#[pliron_op( - name = "volt.va_start", - format = "$0", - interfaces = [NOpdsInterface<1>, NResultsInterface<0>], - verifier = "succ", -)] -pub struct VaStartOp; - -impl VaStartOp { - /// Build a va_start. - pub fn new(ctx: &mut Context, list: Value) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - alloc::vec![list], - Vec::new(), - 0, - ); - Self { op } - } - - /// Argument list pointer. - pub fn list(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Fetch the next variadic argument. -#[pliron_op( - name = "volt.va_arg", - format = "attr($va_arg_ty, $TypeAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (va_arg_ty: TypeAttr), - verifier = "succ", -)] -pub struct VaArgOp; - -impl VaArgOp { - /// Build a va_arg. - pub fn new(ctx: &mut Context, list: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![list], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_va_arg_ty(ctx, TypeAttr::new(ty)); - op - } - - /// Argument list pointer. - pub fn list(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// End of variadic argument access. -#[pliron_op( - name = "volt.va_end", - format = "$0", - interfaces = [NOpdsInterface<1>, NResultsInterface<0>], - verifier = "succ", -)] -pub struct VaEndOp; - -impl VaEndOp { - /// Build a va_end. - pub fn new(ctx: &mut Context, list: Value) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - alloc::vec![list], - Vec::new(), - 0, - ); - Self { op } - } - - /// Argument list pointer. - pub fn list(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Fused dot product: `acc + dot(a, b)`. -#[pliron_op( - name = "volt.dot", - format = "$0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct DotOp; - -impl DotOp { - /// Build a dot product. - pub fn new(ctx: &mut Context, acc: Value, a: Value, b: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![acc, a, b], - Vec::new(), - 0, - ); - Self { op } - } - - /// Accumulator and linear-algebra operands. - pub fn operands3(&self, ctx: &Context) -> (Value, Value, Value) { - let op = self.get_operation().deref(ctx); - (op.get_operand(0), op.get_operand(1), op.get_operand(2)) - } -} - -/// Vector reduction with a binary operator. -#[pliron_op( - name = "volt.reduce", - format = "attr($reduce_kind, $BinOpAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (reduce_kind: BinOpAttr), - verifier = "succ", -)] -pub struct ReduceOp; - -impl ReduceOp { - /// Build a reduction. - pub fn new(ctx: &mut Context, kind: BinOp, vector: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![vector], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_reduce_kind(ctx, BinOpAttr::new(kind)); - op - } - - /// The reduction operator. - pub fn kind(&self, ctx: &Context) -> BinOp { - req_attr(self.get_attr_reduce_kind(ctx), "volt.reduce", "reduce_kind").value() - } - - /// The reduced vector. - pub fn vector(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Broadcast a scalar to a vector. -#[pliron_op( - name = "volt.splat", - format = "$0 ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct SplatOp; - -impl SplatOp { - /// Build a splat. - pub fn new(ctx: &mut Context, scalar: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![scalar], - Vec::new(), - 0, - ); - Self { op } - } - - /// The broadcast scalar. - pub fn scalar(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Matrix multiply-accumulate over three buffers. -#[pliron_op( - name = "volt.matmul", - format = "attr($matmul_config, $MatMulAttr) ` ` $0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (matmul_config: MatMulAttr), - verifier = "succ", -)] -pub struct MatMulOp; - -impl MatMulOp { - /// Build a matmul. - pub fn new( - ctx: &mut Context, - a: Value, - b: Value, - c: Value, - config: MatMulDesc, - ty: TypeHandle, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![a, b, c], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_matmul_config(ctx, MatMulAttr::new(config)); - op - } - - /// Configuration. - pub fn config(&self, ctx: &Context) -> MatMulDesc { - req_attr( - self.get_attr_matmul_config(ctx), - "volt.matmul", - "matmul_config", - ) - .desc() - .clone() - } - - /// Buffer operands (a, b, c). - pub fn operands3(&self, ctx: &Context) -> (Value, Value, Value) { - let op = self.get_operation().deref(ctx); - (op.get_operand(0), op.get_operand(1), op.get_operand(2)) - } -} - -/// Execution barrier. -#[pliron_op( - name = "volt.barrier", - format = "attr($barrier_scope, $BarrierAttr)", - interfaces = [NOpdsInterface<0>, NResultsInterface<0>], - attributes = (barrier_scope: BarrierAttr), - verifier = "succ", -)] -pub struct BarrierOp; - -impl BarrierOp { - /// Build a barrier. - pub fn new(ctx: &mut Context, scope: BarrierScope) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_barrier_scope(ctx, BarrierAttr::new(scope)); - op - } - - /// Barrier scope. - pub fn scope(&self, ctx: &Context) -> BarrierScope { - req_attr( - self.get_attr_barrier_scope(ctx), - "volt.barrier", - "barrier_scope", - ) - .value() - } -} - -/// Atomic read-modify-write. Operands are `ptr, value`, plus `compare` -/// for compare-exchange. -#[pliron_op( - name = "volt.atomic_rmw", - format = "attr($atomic_kind, $AtomicOpAttr) ` ` attr($atomic_ordering, $AtomicOrderingAttr) ` ` attr($atomic_scope, $AtomicScopeAttr) ` ` operands(CharSpace(`,`)) ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (atomic_kind: AtomicOpAttr, atomic_ordering: AtomicOrderingAttr, atomic_scope: AtomicScopeAttr), - verifier = "succ", -)] -pub struct AtomicRmwOp; - -impl AtomicRmwOp { - /// Build an atomic RMW. - pub fn new( - ctx: &mut Context, - kind: AtomicOp, - ptr: Value, - value: Value, - compare: Option, - ordering: AtomicOrdering, - scope: AtomicScope, - ty: TypeHandle, - ) -> Self { - let mut operands = alloc::vec![ptr, value]; - if let Some(c) = compare { - operands.push(c); - } - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - operands, - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_atomic_kind(ctx, AtomicOpAttr::new(kind)); - op.set_attr_atomic_ordering(ctx, AtomicOrderingAttr::new(ordering)); - op.set_attr_atomic_scope(ctx, AtomicScopeAttr::new(scope)); - op - } - - /// The atomic operator. - pub fn kind(&self, ctx: &Context) -> AtomicOp { - req_attr( - self.get_attr_atomic_kind(ctx), - "volt.atomic_rmw", - "atomic_kind", - ) - .value() - } - - /// Ordering. - pub fn ordering(&self, ctx: &Context) -> AtomicOrdering { - req_attr( - self.get_attr_atomic_ordering(ctx), - "volt.atomic_rmw", - "atomic_ordering", - ) - .value() - } - - /// Scope. - pub fn scope(&self, ctx: &Context) -> AtomicScope { - req_attr( - self.get_attr_atomic_scope(ctx), - "volt.atomic_rmw", - "atomic_scope", - ) - .value() - } - - /// Pointer, value, and optional compare operands. - pub fn operands_rmw(&self, ctx: &Context) -> (Value, Value, Option) { - let op = self.get_operation().deref(ctx); - let n = op.get_num_operands(); - ( - op.get_operand(0), - op.get_operand(1), - if n > 2 { Some(op.get_operand(2)) } else { None }, - ) - } -} - -/// Unconditional branch: `volt.jump ^target(%args)`. -#[pliron_op( - name = "volt.jump", - format = "succ($0) `(` operands(CharSpace(`,`)) `)`", - interfaces = [IsTerminatorInterface, NResultsInterface<0>], - verifier = "succ", -)] -pub struct JumpOp; - -impl JumpOp { - /// Build a jump. - pub fn new(ctx: &mut Context, target: Ptr, args: Vec) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - args, - alloc::vec![target], - 0, - ); - Self { op } - } - - /// Jump target. - pub fn target(&self, ctx: &Context) -> Ptr { - self.get_operation().deref(ctx).get_successor(0) - } - - /// Block arguments. - pub fn args(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().collect() - } -} - -/// Conditional branch with per-successor operands. -/// -/// Stored as one successor list `[then, else]`, one operand list -/// `[cond, ...then_args, ...else_args]`, and a `then_args` count -/// attribute splitting the two argument groups. -/// -/// Syntax: `volt.cond_br %cond, ^then(%a), ^else(%b)`. -#[pliron_op( - name = "volt.cond_br", - interfaces = [IsTerminatorInterface, NResultsInterface<0>], - attributes = (cond_then_args: ImmAttr), - verifier = "succ", -)] -pub struct CondBrOp; - -impl CondBrOp { - /// Build a conditional branch. - pub fn new( - ctx: &mut Context, - cond: Value, - then: Ptr, - then_args: Vec, - else_: Ptr, - else_args: Vec, - ) -> Self { - let n_then = then_args.len() as i64; - let mut operands = alloc::vec![cond]; - operands.extend(then_args); - operands.extend(else_args); - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - operands, - alloc::vec![then, else_], - 0, - ); - let op = Self { op }; - op.set_attr_cond_then_args(ctx, ImmAttr::new(n_then)); - op - } - - /// Branch condition. - pub fn cond(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Number of then-successor arguments. - fn n_then(&self, ctx: &Context) -> usize { - req_attr( - self.get_attr_cond_then_args(ctx), - "volt.cond_br", - "cond_then_args", - ) - .value() as usize - } - - /// Then destination. - pub fn then_dest(&self, ctx: &Context) -> Ptr { - self.get_operation().deref(ctx).get_successor(0) - } - - /// Else destination. - pub fn else_dest(&self, ctx: &Context) -> Ptr { - self.get_operation().deref(ctx).get_successor(1) - } - - /// Then arguments. - pub fn then_args(&self, ctx: &Context) -> Vec { - let n = self.n_then(ctx); - self.get_operation() - .deref(ctx) - .operands() - .skip(1) - .take(n) - .collect() - } - - /// Else arguments. - pub fn else_args(&self, ctx: &Context) -> Vec { - let n = self.n_then(ctx); - self.get_operation() - .deref(ctx) - .operands() - .skip(1 + n) - .collect() - } -} - -impl Printable for CondBrOp { - fn fmt( - &self, - ctx: &Context, - state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - write!( - f, - "{} {}, ^{}({}), ^{}({})", - self.get_opid().print(ctx, state), - self.cond(ctx).print(ctx, state), - self.then_dest(ctx).unique_name(ctx), - comma_values(ctx, state, &self.then_args(ctx)), - self.else_dest(ctx).unique_name(ctx), - comma_values(ctx, state, &self.else_args(ctx)), - ) - } -} - -fn comma_values(ctx: &Context, state: &printable::State, values: &[Value]) -> String { - let mut out = String::new(); - for (i, v) in values.iter().enumerate() { - if i != 0 { - out.push_str(", "); - } - out.push_str(&alloc::format!("{}", v.print(ctx, state))); - } - out -} - -impl Parsable for CondBrOp { - type Arg = Vec<(Identifier, Location)>; - type Parsed = OpObj; - - fn parse<'a>( - state_stream: &mut StateStream<'a>, - results: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - let loc = state_stream.loc(); - if !results.is_empty() { - input_err!(loc, "volt.cond_br cannot define SSA values")?; - } - let cond = ssa_opd_parser().parse_stream(state_stream).into_result()?.0; - spaced(token(',')) - .parse_stream(state_stream) - .into_result()?; - let then = block_opd_parser() - .parse_stream(state_stream) - .into_result()? - .0; - let then_args = delimited_list_parser('(', ')', ',', ssa_opd_parser()) - .parse_stream(state_stream) - .into_result()? - .0; - spaced(token(',')) - .parse_stream(state_stream) - .into_result()?; - let else_ = block_opd_parser() - .parse_stream(state_stream) - .into_result()? - .0; - let else_args = delimited_list_parser('(', ')', ',', ssa_opd_parser()) - .parse_stream(state_stream) - .into_result()? - .0; - let op = Self::new( - state_stream.state.ctx, - cond, - then, - then_args, - else_, - else_args, - ); - process_parsed_ssa_defs(state_stream, &results, op.get_operation())?; - Ok(OpObj::new(op)).into_parse_result() - } -} - -/// Function return with up to N values. -#[pliron_op( - name = "volt.ret", - format = "operands(CharSpace(`,`))", - interfaces = [IsTerminatorInterface, NResultsInterface<0>], - verifier = "succ", -)] -pub struct RetOp; - -impl RetOp { - /// Build a return. - pub fn new(ctx: &mut Context, values: Vec) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - values, - Vec::new(), - 0, - ); - Self { op } - } - - /// Returned values. - pub fn values(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::dialect::types::{BoolType, IntType}; - use pliron::{ - builtin::{op_interfaces::OneRegionInterface, ops::FuncOp}, - parsable::parse_from_str, - result::ExpectOk, - }; - - #[test] - fn arith_print_parse() { - let ctx = &mut Context::new(); - let i32t = IntType::get_type(ctx, true, 32); - let boolt = BoolType::get_type(ctx); - let func_ty = pliron::builtin::types::FunctionType::get(ctx, vec![i32t, i32t], vec![boolt]); - let func = FuncOp::new(ctx, pliron::ident!("f"), func_ty); - let entry = func.get_entry_block(ctx); - let a = entry.deref(ctx).get_argument(0); - let b = entry.deref(ctx).get_argument(1); - let add = ArithOp::new(ctx, BinOp::Add, a, b, i32t); - add.get_operation().insert_at_back(entry, ctx); - let text = alloc::format!( - "{}", - add.get_operation() - .deref(ctx) - .print(ctx, &Default::default()) - ); - assert!(text.contains("volt.arith"), "{text}"); - assert!(text.contains("add"), "{text}"); - let _ = boolt; - } - - #[test] - fn cond_br_round_trip() { - let ctx = &mut Context::new(); - let boolt = BoolType::get_type(ctx); - let func_ty = pliron::builtin::types::FunctionType::get(ctx, vec![boolt], vec![]); - let func = FuncOp::new(ctx, pliron::ident!("f"), func_ty); - let entry = func.get_entry_block(ctx); - let region = func.get_region(ctx); - let then_bb = BasicBlock::new(ctx, Some(pliron::ident!("then")), vec![]); - then_bb.insert_at_back(region, ctx); - let else_bb = BasicBlock::new(ctx, Some(pliron::ident!("else")), vec![]); - else_bb.insert_at_back(region, ctx); - let cond = entry.deref(ctx).get_argument(0); - let br = CondBrOp::new(ctx, cond, then_bb, vec![], else_bb, vec![]); - br.get_operation().insert_at_back(entry, ctx); - let func_text = alloc::format!( - "{}", - func.get_operation() - .deref(ctx) - .print(ctx, &Default::default()) - ); - let parsed = parse_from_str(Operation::top_level_parser(), ctx, &func_text).expect_ok(ctx); - let _ = parsed; - } -} diff --git a/src/ir/src/dialect/types.rs b/src/ir/src/dialect/types.rs deleted file mode 100644 index cae2eeb..0000000 --- a/src/ir/src/dialect/types.rs +++ /dev/null @@ -1,450 +0,0 @@ -//! The volt type system as pliron types. -//! -//! Every variant of the legacy [`TypeKind`](crate::types::TypeKind) has a -//! corresponding pliron type in the `volt` dialect: -//! -//! | volt type | example syntax | -//! |------------------------|-----------------------------------------| -//! | [`BoolType`] | `volt.bool` | -//! | [`IntType`] | `volt.int` | -//! | [`FloatType`] | `volt.float` | -//! | [`PtrType`] | `volt.ptr(global)` | -//! | [`VectorType`] | `volt.vector<4 x volt.int>` | -//! | [`ArrayType`] | `volt.array[8 x volt.float]` | -//! | [`SliceType`] | `volt.slice[]volt.int` | -//! | [`StructType`] | `volt.struct{...}` | -//! -//! [`describe`] inspects any [`TypeHandle`] and returns an owned -//! [`VoltKind`], which is the main entry point for instruction selection. - -use alloc::vec::Vec; -use core::hash::Hash; - -use pliron::{ - combine::{ - Parser, attempt, choice, - parser::char::{spaces, string}, - parser::token::token, - }, - context::Context, - derive::pliron_type, - parsable::{Parsable, ParseResult, StateStream}, - printable::{self, Printable}, - r#type::TypeHandle, -}; - -use super::super::types::{AddressSpace, FloatKind}; - -/// A boolean value. -#[pliron_type(name = "volt.bool", generate_get = true, format, verifier = "succ")] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy, Default)] -pub struct BoolType; - -impl BoolType { - /// Intern (or fetch) the boolean type. - pub fn get_type(ctx: &Context) -> TypeHandle { - Self::get(ctx).into() - } -} - -/// A fixed-width integer. `signed` selects the `i`/`u` flavor. -#[pliron_type( - name = "volt.int", - generate_get = true, - format = "`<` $signed `, ` $bits `>`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct IntType { - signed: bool, - bits: u32, -} - -impl IntType { - /// Intern (or fetch) an integer type. - pub fn get_type(ctx: &Context, signed: bool, bits: u32) -> TypeHandle { - Self::get(ctx, signed, bits).into() - } - - /// Is this the signed flavor? - pub fn is_signed(&self) -> bool { - self.signed - } - - /// Bit width. - pub fn bits(&self) -> u32 { - self.bits - } -} - -/// A floating-point scalar. Prints with the legacy keyword (`f32`, ...). -#[pliron_type( - name = "volt.float", - generate_get = true, - format = "`<` $kind `>`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct FloatType { - kind: FloatKind, -} - -impl FloatType { - /// Intern (or fetch) a float type. - pub fn get_type(ctx: &Context, kind: FloatKind) -> TypeHandle { - Self::get(ctx, kind).into() - } - - /// The float flavor. - pub fn kind(&self) -> FloatKind { - self.kind - } -} - -/// A pointer into an address space. -#[pliron_type( - name = "volt.ptr", - generate_get = true, - format = "`(` $space `)`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct PtrType { - space: AddressSpace, -} - -impl PtrType { - /// Intern (or fetch) a pointer type. - pub fn get_type(ctx: &Context, space: AddressSpace) -> TypeHandle { - Self::get(ctx, space).into() - } - - /// The address space. - pub fn space(&self) -> AddressSpace { - self.space - } -} - -/// A fixed-length SIMD vector. -#[pliron_type( - name = "volt.vector", - generate_get = true, - format = "`<` $len ` x ` $elem `>`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct VectorType { - len: u32, - elem: TypeHandle, -} - -impl VectorType { - /// Intern (or fetch) a vector type. - pub fn get_type(ctx: &Context, len: u32, elem: TypeHandle) -> TypeHandle { - Self::get(ctx, len, elem).into() - } - - /// Lane count. - pub fn len(&self) -> u32 { - self.len - } - - /// Lane type. - pub fn elem(&self) -> TypeHandle { - self.elem - } -} - -/// A fixed-length array. -#[pliron_type( - name = "volt.array", - generate_get = true, - format = "`[` $len ` x ` $elem `]`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct ArrayType { - len: u64, - elem: TypeHandle, -} - -impl ArrayType { - /// Intern (or fetch) an array type. - pub fn get_type(ctx: &Context, len: u64, elem: TypeHandle) -> TypeHandle { - Self::get(ctx, len, elem).into() - } - - /// Element count. - pub fn len(&self) -> u64 { - self.len - } - - /// Element type. - pub fn elem(&self) -> TypeHandle { - self.elem - } -} - -/// An unsized slice (fat pointer payload). -#[pliron_type( - name = "volt.slice", - generate_get = true, - format = "`[` `]` $elem", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct SliceType { - elem: TypeHandle, -} - -impl SliceType { - /// Intern (or fetch) a slice type. - pub fn get_type(ctx: &Context, elem: TypeHandle) -> TypeHandle { - Self::get(ctx, elem).into() - } - - /// Element type. - pub fn elem(&self) -> TypeHandle { - self.elem - } -} - -/// A heterogeneous struct. -#[pliron_type( - name = "volt.struct", - generate_get = true, - format = "`{` vec($fields, CharSpace(`,`)) `}`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone)] -pub struct StructType { - fields: Vec, -} - -impl StructType { - /// Intern (or fetch) a struct type. - pub fn get_type(ctx: &Context, fields: Vec) -> TypeHandle { - Self::get(ctx, fields).into() - } - - /// Field types. - pub fn fields(&self) -> &[TypeHandle] { - &self.fields - } -} - -/// An owned description of any volt type, mirroring the legacy `TypeKind`. -/// This is the main inspection entry point for instruction selection. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum VoltKind { - /// `volt.bool`. - Bool, - /// `volt.int`. - Int { signed: bool, bits: u32 }, - /// `volt.float`. - Float(FloatKind), - /// `volt.ptr`. - Ptr(AddressSpace), - /// `volt.vector`. - Vector { len: u32, elem: TypeHandle }, - /// `volt.array`. - Array { len: u64, elem: TypeHandle }, - /// `volt.slice`. - Slice { elem: TypeHandle }, - /// `volt.struct`. - Struct(Vec), -} - -/// Describe a type handle as an owned [`VoltKind`]. Returns `None` for -/// non-volt types (e.g. `builtin.function`). -pub fn describe(ctx: &Context, ty: TypeHandle) -> Option { - let obj = ty.deref(ctx); - if let Some(t) = obj.downcast_ref::() { - let _ = t; - return Some(VoltKind::Bool); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Int { - signed: t.is_signed(), - bits: t.bits(), - }); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Float(t.kind())); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Ptr(t.space())); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Vector { - len: t.len(), - elem: t.elem(), - }); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Array { - len: t.len(), - elem: t.elem(), - }); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Slice { elem: t.elem() }); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Struct(t.fields().to_vec())); - } - None -} - -/// Intern an owned [`VoltKind`] description. -pub fn intern(ctx: &Context, kind: &VoltKind) -> TypeHandle { - match kind { - VoltKind::Bool => BoolType::get_type(ctx), - VoltKind::Int { signed, bits } => IntType::get_type(ctx, *signed, *bits), - VoltKind::Float(k) => FloatType::get_type(ctx, *k), - VoltKind::Ptr(s) => PtrType::get_type(ctx, *s), - VoltKind::Vector { len, elem } => VectorType::get_type(ctx, *len, *elem), - VoltKind::Array { len, elem } => ArrayType::get_type(ctx, *len, *elem), - VoltKind::Slice { elem } => SliceType::get_type(ctx, *elem), - VoltKind::Struct(f) => StructType::get_type(ctx, f.clone()), - } -} - -// The `$kind` / `$space` format fields above need print/parse support for -// the legacy descriptor enums. These print with the legacy keywords. - -impl Printable for FloatKind { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let s = match self { - FloatKind::F32 => "f32", - FloatKind::F64 => "f64", - FloatKind::F16 => "f16", - FloatKind::F128 => "f128", - }; - f.write_str(s) - } -} - -impl Printable for AddressSpace { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let s = match self { - AddressSpace::Global => "global", - AddressSpace::Shared => "shared", - AddressSpace::Private => "private", - AddressSpace::Constant => "constant", - }; - f.write_str(s) - } -} - -impl Parsable for FloatKind { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(token('f').with(choice(( - string("32").map(|_| FloatKind::F32), - string("64").map(|_| FloatKind::F64), - attempt(string("128")).map(|_| FloatKind::F128), - string("16").map(|_| FloatKind::F16), - )))) - .parse_stream(state_stream) - .into() - } -} - -impl Parsable for AddressSpace { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(choice(( - string("global").map(|_| AddressSpace::Global), - string("shared").map(|_| AddressSpace::Shared), - string("private").map(|_| AddressSpace::Private), - string("constant").map(|_| AddressSpace::Constant), - ))) - .parse_stream(state_stream) - .into() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pliron::{ - parsable::{Parsable, parse_from_str}, - result::ExpectOk, - }; - - fn ctx() -> Context { - Context::new() - } - - #[test] - fn int_type_round_trip() { - let ctx = &mut ctx(); - let ty = IntType::get_type(ctx, true, 32); - assert_eq!( - describe(ctx, ty), - Some(VoltKind::Int { - signed: true, - bits: 32 - }) - ); - let parsed = - parse_from_str(TypeHandle::parser(()), ctx, "volt.int").expect_ok(ctx); - assert_eq!(parsed, ty); - } - - #[test] - fn composite_type_round_trip() { - let ctx = &mut ctx(); - let i32t = IntType::get_type(ctx, true, 32); - let v = VectorType::get_type(ctx, 4, i32t); - let printed = alloc::format!("{}", v.print(ctx, &Default::default())); - let back = parse_from_str(TypeHandle::parser(()), ctx, &printed).expect_ok(ctx); - assert_eq!(back, v); - - let s = StructType::get_type(ctx, alloc::vec![i32t, BoolType::get_type(ctx)]); - let printed = alloc::format!("{}", s.print(ctx, &Default::default())); - let back = parse_from_str(TypeHandle::parser(()), ctx, &printed).expect_ok(ctx); - assert_eq!(back, s); - } - - #[test] - fn float_ptr_kinds() { - let ctx = &mut ctx(); - for (text, kind) in [ - ("volt.float", VoltKind::Float(FloatKind::F32)), - ("volt.float", VoltKind::Float(FloatKind::F128)), - ("volt.ptr(shared)", VoltKind::Ptr(AddressSpace::Shared)), - ( - "volt.array[8 x volt.float]", - VoltKind::Array { - len: 8, - elem: FloatType::get_type(ctx, FloatKind::F64), - }, - ), - ] { - let parsed = parse_from_str(TypeHandle::parser(()), ctx, text).expect_ok(ctx); - assert_eq!(describe(ctx, parsed), Some(kind), "{text}"); - } - } -} diff --git a/src/ir/src/expand.rs b/src/ir/src/expand.rs index 1cf498b..b38a306 100644 --- a/src/ir/src/expand.rs +++ b/src/ir/src/expand.rs @@ -4,8 +4,8 @@ use alloc::vec::Vec; use super::function::AttrTarget; use super::function::{ - BinOp, Block, CmpOp, Function, Inst, LowFloatConvert, MatMul, MatMulType, NvFp4Convert, Opcode, - Value, + BinOp, Block, CmpOp, ConvertKind, Function, Inst, LowFloatConvert, MatMul, MatMulType, + MemFlags, NvFp4Convert, Opcode, Value, }; use super::low_float::Format as LowFloatFormat; use super::nvfp4::ScaleApplication; @@ -138,19 +138,22 @@ pub fn expand_vector_lanes_except( // --- NumericBuilder ---------------------------------------------------------- struct NumericBuilder { - u8_t: Type, - u32_t: Type, + i8_t: Type, + i32_t: Type, f32_t: Type, bool_t: Type, out: Vec, } /// The bounded operations a scalar expansion may emit. +/// +/// Every integer here is a plain 32-bit bit pattern read as UNSIGNED: widening zero-extends, +/// right shifts are logical and magnitude compares are `Ult`/`Ugt`/.... impl NumericBuilder { fn common(_func: &mut Function, ct: &CommonTypes) -> Self { Self { - u8_t: ct.u8_t, - u32_t: ct.u32_t, + i8_t: ct.i8_t, + i32_t: ct.i32_t, f32_t: ct.f32_t, bool_t: ct.bool_t, out: Vec::new(), @@ -164,13 +167,13 @@ impl NumericBuilder { } fn constant(&mut self, func: &mut Function, value: u32) -> Value { - self.emit(func, self.u32_t, Opcode::Iconst(value as i64)) + self.emit(func, self.i32_t, Opcode::Iconst(value as i64)) } fn binary(&mut self, func: &mut Function, op: BinOp, lhs: Value, rhs: Value) -> Value { self.emit( func, - self.u32_t, + self.i32_t, Opcode::Arith(super::function::Arith { op, lhs, rhs }), ) } @@ -197,7 +200,7 @@ impl NumericBuilder { fn select(&mut self, func: &mut Function, cond: Value, then_v: Value, else_v: Value) -> Value { self.emit( func, - self.u32_t, + self.i32_t, Opcode::Select(super::function::Select { cond, then: then_v, @@ -206,14 +209,31 @@ impl NumericBuilder { ) } - fn convert(&mut self, func: &mut Function, ty: Type, value: Value) -> Value { + fn convert(&mut self, func: &mut Function, ty: Type, kind: ConvertKind, value: Value) -> Value { self.emit( func, ty, - Opcode::Convert(super::function::Convert { value }), + Opcode::Convert(super::function::Convert { value, kind }), ) } + /// `value` as the i32 holding its bits zero-extended: a narrower int (or bool) is widened + /// with `Zext`, a wider one narrowed with `Trunc`, and an i32 is used as it is. + fn as_i32(&mut self, func: &mut Function, value: Value) -> Value { + let bits = match func.types.type_kind(func.value_type(value)) { + TypeKind::Int(d) => u32::from(d.bits), + TypeKind::Bool => 1, + _ => return value, + }; + match bits.cmp(&32) { + core::cmp::Ordering::Less => self.convert(func, self.i32_t, ConvertKind::Zext, value), + core::cmp::Ordering::Equal => value, + core::cmp::Ordering::Greater => { + self.convert(func, self.i32_t, ConvertKind::Trunc, value) + } + } + } + fn reinterpret(&mut self, func: &mut Function, ty: Type, value: Value) -> Value { self.emit( func, @@ -249,7 +269,7 @@ impl NumericBuilder { let discarded = self.binary(func, BinOp::BitAnd, value, mask); let shift_minus_one = self.binary(func, BinOp::Sub, shift, one); let halfway = self.binary(func, BinOp::Shl, one, shift_minus_one); - let greater = self.compare(func, CmpOp::Gt, discarded, halfway); + let greater = self.compare(func, CmpOp::Ugt, discarded, halfway); let equal = self.compare(func, CmpOp::Eq, discarded, halfway); let odd_bits = self.binary(func, BinOp::BitAnd, retained, one); let zero = self.constant(func, 0); @@ -265,30 +285,30 @@ impl NumericBuilder { /// Types used by the scalar builder passes, interned once per pass run. struct CommonTypes { - u8_t: Type, - u16_t: Type, - u32_t: Type, + i8_t: Type, + i16_t: Type, + i32_t: Type, f32_t: Type, bool_t: Type, } fn intern_common(func: &mut Function) -> CommonTypes { - let u8_t = func.types.intern(int_ty_kind(8, false)); - let u16_t = func.types.intern(int_ty_kind(16, false)); - let u32_t = func.types.intern(int_ty_kind(32, false)); + let i8_t = func.types.intern(int_ty_kind(8)); + let i16_t = func.types.intern(int_ty_kind(16)); + let i32_t = func.types.intern(int_ty_kind(32)); let f32_t = func.types.intern(TypeKind::Float(FloatKind::F32)); let bool_t = func.types.intern(TypeKind::Bool); CommonTypes { - u8_t, - u16_t, - u32_t, + i8_t, + i16_t, + i32_t, f32_t, bool_t, } } -fn int_ty_kind(bits: u16, signed: bool) -> TypeKind { - TypeKind::Int(IntDesc { signed, bits }) +fn int_ty_kind(bits: u16) -> TypeKind { + TypeKind::Int(IntDesc { bits }) } /// Replace all low-float storage conversions with target-independent integer /// operations. Returns whether anything changed. @@ -352,7 +372,7 @@ pub fn expand_low_float(func: &mut Function) -> bool { continue; }; let expanded = - encode_low_float(func, &mut builder, &conversion, inst, ct.u16_t, ct.u8_t); + encode_low_float(func, &mut builder, &conversion, inst, ct.i16_t, ct.i8_t); func.replace_all_uses(old_result, expanded); func.retarget_attrs(AttrTarget::Value(old_result), AttrTarget::Value(expanded)); out.append(&mut builder.out); @@ -366,7 +386,7 @@ pub fn expand_low_float(func: &mut Function) -> bool { } fn decode_low_float(func: &mut Function, b: &mut NumericBuilder, conv: &LowFloatConvert) -> Value { - let payload = b.convert(func, b.u32_t, conv.value); + let payload = b.as_i32(func, conv.value); let bits = match conv.format { LowFloatFormat::Bf16 => b.shifted(func, BinOp::Shl, payload, 16), LowFloatFormat::F8E4M3 => decode_e4m3(func, b, payload), @@ -466,10 +486,10 @@ fn encode_low_float( b: &mut NumericBuilder, conv: &LowFloatConvert, old_inst: Inst, - u16_t: Type, - u8_t: Type, + i16_t: Type, + i8_t: Type, ) -> Value { - let bits = b.reinterpret(func, b.u32_t, conv.value); + let bits = b.reinterpret(func, b.i32_t, conv.value); let defining = func.defining_inst(bits).unwrap(); func.retarget_attrs(AttrTarget::Inst(old_inst), AttrTarget::Inst(defining)); let payload = match conv.format { @@ -478,11 +498,11 @@ fn encode_low_float( LowFloatFormat::F8E5M2 => encode_fp8(func, b, bits, false), }; let narrow_ty = if conv.format == LowFloatFormat::Bf16 { - u16_t + i16_t } else { - u8_t + i8_t }; - b.convert(func, narrow_ty, payload) + b.convert(func, narrow_ty, ConvertKind::Trunc, payload) } fn encode_bf16(func: &mut Function, b: &mut NumericBuilder, bits: Value) -> Value { @@ -516,11 +536,11 @@ fn encode_fp8(func: &mut Function, b: &mut NumericBuilder, bits: Value, e4m3: bo let bias_v = b.constant(func, shift_bias); let original_shift = b.binary(func, BinOp::Sub, bias_v, source_exponent); let hoist17 = b.constant(func, 24); - let shift_too_large = b.compare(func, CmpOp::Gt, original_shift, hoist17); + let shift_too_large = b.compare(func, CmpOp::Ugt, original_shift, hoist17); let hoist18 = b.constant(func, 24); let bounded_shift = b.select(func, shift_too_large, hoist18, original_shift); let hoist1 = b.constant(func, threshold); - let subnormal_path = b.compare(func, CmpOp::Lt, source_exponent, hoist1); + let subnormal_path = b.compare(func, CmpOp::Ult, source_exponent, hoist1); let hoist19 = b.constant(func, 24); let safe_shift = b.select(func, subnormal_path, bounded_shift, hoist19); let subnormal_rounded = b.round_right(func, significand, safe_shift); @@ -549,7 +569,7 @@ fn encode_fp8(func: &mut Function, b: &mut NumericBuilder, bits: Value, e4m3: bo let finite_magnitude = b.select(func, subnormal_path, subnormal, normal); let max_finite: u32 = if e4m3 { 0x7e } else { 0x7b }; let hoist4 = b.constant(func, max_finite); - let too_large = b.compare(func, CmpOp::Gt, finite_magnitude, hoist4); + let too_large = b.compare(func, CmpOp::Ugt, finite_magnitude, hoist4); let hoist22 = b.constant(func, max_finite); let saturated = b.select(func, too_large, hoist22, finite_magnitude); let finite = b.binary(func, BinOp::BitOr, sign, saturated); @@ -686,7 +706,7 @@ fn dequantize_nvfp4( b: &mut NumericBuilder, conv: &NvFp4Convert, ) -> NvFp4Expansion { - let carrier = b.convert(func, b.u32_t, conv.value); + let carrier = b.as_i32(func, conv.value); let nibble = b.masked(func, carrier, 0x0f); let sign_part = b.masked(func, nibble, 0x08); let sign = b.shifted(func, BinOp::Shl, sign_part, 28); @@ -711,7 +731,7 @@ fn dequantize_nvfp4( let value_bits = b.binary(func, BinOp::BitOr, sign, magnitude); let value = b.reinterpret(func, b.f32_t, value_bits); - let block_payload = b.convert(func, b.u32_t, conv.block_scale); + let block_payload = b.as_i32(func, conv.block_scale); let block_bits = decode_e4m3(func, b, block_payload); let block_scale = b.reinterpret(func, b.f32_t, block_bits); let local = b.float_binary( @@ -730,7 +750,7 @@ fn dequantize_nvfp4( // This bit round trip blocks a backend from fusing the final scale with an // original consumer. - let scaled_bits = b.reinterpret(func, b.u32_t, scaled); + let scaled_bits = b.reinterpret(func, b.i32_t, scaled); let result = b.reinterpret(func, b.f32_t, scaled_bits); NvFp4Expansion { result, @@ -743,7 +763,7 @@ fn quantize_nvfp4( b: &mut NumericBuilder, conv: &NvFp4Convert, ) -> NvFp4Expansion { - let block_payload = b.convert(func, b.u32_t, conv.block_scale); + let block_payload = b.as_i32(func, conv.block_scale); let block_bits = decode_e4m3(func, b, block_payload); let block_scale = b.reinterpret(func, b.f32_t, block_bits); let global_unscaled = b.float_binary( @@ -758,7 +778,7 @@ fn quantize_nvfp4( global_unscaled, block_scale, ); - let bits = b.reinterpret(func, b.u32_t, local_unscaled); + let bits = b.reinterpret(func, b.i32_t, local_unscaled); let semantic_inst = func.defining_inst(bits).unwrap(); let shifted_bits = b.shifted(func, BinOp::Shr, bits, 28); @@ -766,13 +786,13 @@ fn quantize_nvfp4( let magnitude = b.masked(func, bits, 0x7fff_ffff); let mut payload = b.constant(func, 0x07); for (relation, threshold_bits, threshold_payload) in [ - (CmpOp::Le, 0x40a0_0000u32, 0x06u32), - (CmpOp::Lt, 0x4060_0000, 0x05), - (CmpOp::Le, 0x4020_0000, 0x04), - (CmpOp::Lt, 0x3fe0_0000, 0x03), - (CmpOp::Le, 0x3fa0_0000, 0x02), - (CmpOp::Lt, 0x3f40_0000, 0x01), - (CmpOp::Le, 0x3e80_0000, 0x00), + (CmpOp::Ule, 0x40a0_0000u32, 0x06u32), + (CmpOp::Ult, 0x4060_0000, 0x05), + (CmpOp::Ule, 0x4020_0000, 0x04), + (CmpOp::Ult, 0x3fe0_0000, 0x03), + (CmpOp::Ule, 0x3fa0_0000, 0x02), + (CmpOp::Ult, 0x3f40_0000, 0x01), + (CmpOp::Ule, 0x3e80_0000, 0x00), ] { let hoist29 = b.constant(func, threshold_bits); let cond = b.compare(func, relation, magnitude, hoist29); @@ -781,10 +801,10 @@ fn quantize_nvfp4( } let signed_payload = b.binary(func, BinOp::BitOr, sign, payload); let hoist5 = b.constant(func, 0x7f80_0000); - let is_nan = b.compare(func, CmpOp::Gt, magnitude, hoist5); + let is_nan = b.compare(func, CmpOp::Ugt, magnitude, hoist5); let hoist31 = b.constant(func, 0x07); let canonical = b.select(func, is_nan, hoist31, signed_payload); - let result = b.convert(func, b.u8_t, canonical); + let result = b.convert(func, b.i8_t, ConvertKind::Trunc, canonical); NvFp4Expansion { result, semantic_inst, @@ -813,7 +833,7 @@ fn normalize_f32(func: &mut Function, b: &mut NumericBuilder, magnitude: Value) let mut exponent = exponent; for shift in [16u32, 8, 4, 2, 1] { let threshold_v = b.constant(func, 0x0100_0000u32 >> shift); - let needs_shift = b.compare(func, CmpOp::Lt, significand, threshold_v); + let needs_shift = b.compare(func, CmpOp::Ult, significand, threshold_v); let shifted_sig = b.shifted(func, BinOp::Shl, significand, shift); significand = b.select(func, needs_shift, shifted_sig, significand); let hoist7 = b.constant(func, shift as i64 as u32); @@ -851,7 +871,7 @@ fn round_division_quotient( let discarded = b.binary(func, BinOp::BitAnd, quotient, mask); let shift_minus_one = b.binary(func, BinOp::Sub, shift, one); let halfway = b.binary(func, BinOp::Shl, one, shift_minus_one); - let discarded_gt = b.compare(func, CmpOp::Gt, discarded, halfway); + let discarded_gt = b.compare(func, CmpOp::Ugt, discarded, halfway); let above_half = bool_as_u32(func, b, discarded_gt); let discarded_eq = b.compare(func, CmpOp::Eq, discarded, halfway); let at_half = bool_as_u32(func, b, discarded_eq); @@ -876,8 +896,8 @@ fn expand_f32_division( numerator: Value, denominator: Value, ) -> Value { - let numerator_bits = b.reinterpret(func, b.u32_t, numerator); - let denominator_bits = b.reinterpret(func, b.u32_t, denominator); + let numerator_bits = b.reinterpret(func, b.i32_t, numerator); + let denominator_bits = b.reinterpret(func, b.i32_t, denominator); let numerator_magnitude = b.masked(func, numerator_bits, 0x7fff_ffff); let denominator_magnitude = b.masked(func, denominator_bits, 0x7fff_ffff); let signs = b.binary(func, BinOp::BitXor, numerator_bits, denominator_bits); @@ -887,7 +907,7 @@ fn expand_f32_division( let ratio_lt = b.compare( func, - CmpOp::Lt, + CmpOp::Ult, normalized_numerator.significand, normalized_denominator.significand, ); @@ -915,7 +935,7 @@ fn expand_f32_division( for step in 0..27 { let bit = b.compare( func, - CmpOp::Ge, + CmpOp::Uge, remainder, normalized_denominator.significand, ); @@ -932,12 +952,12 @@ fn expand_f32_division( } let hh1 = b.constant(func, 256); - let exp_le = b.compare(func, CmpOp::Le, result_exponent, hh1); + let exp_le = b.compare(func, CmpOp::Ule, result_exponent, hh1); let is_tiny = bool_as_u32(func, b, exp_le); let hh2 = b.constant(func, 260); let raw_tiny_shift = b.binary(func, BinOp::Sub, hh2, result_exponent); let hh3 = b.constant(func, 31); - let shift_gt = b.compare(func, CmpOp::Gt, raw_tiny_shift, hh3); + let shift_gt = b.compare(func, CmpOp::Ugt, raw_tiny_shift, hh3); let shift_too_large = bool_as_u32(func, b, shift_gt); let shift_large_set = flag_set(func, b, shift_too_large); let hh4 = b.constant(func, 31); @@ -948,7 +968,7 @@ fn expand_f32_division( let rounded = round_division_quotient(func, b, quotient, remainder, rounding_shift); let hh6 = b.constant(func, 0x0100_0000); - let carry_ge = b.compare(func, CmpOp::Ge, rounded, hh6); + let carry_ge = b.compare(func, CmpOp::Uge, rounded, hh6); let rounded_carry = bool_as_u32(func, b, carry_ge); let carry_set = flag_set(func, b, rounded_carry); let halved = b.shifted(func, BinOp::Shr, rounded, 1); @@ -962,10 +982,10 @@ fn expand_f32_division( let tiny_set2 = flag_set(func, b, is_tiny); let finite_bits = b.select(func, tiny_set2, rounded, normal_bits); let hh8 = b.constant(func, 511); - let overflow_before = b.compare(func, CmpOp::Ge, result_exponent, hh8); + let overflow_before = b.compare(func, CmpOp::Uge, result_exponent, hh8); let overflow_before = bool_as_u32(func, b, overflow_before); let hh9 = b.constant(func, 255); - let overflow_after = b.compare(func, CmpOp::Ge, normal_exponent, hh9); + let overflow_after = b.compare(func, CmpOp::Uge, normal_exponent, hh9); let overflow_after = bool_as_u32(func, b, overflow_after); let tiny_set3 = flag_set(func, b, is_tiny); let hh10 = b.constant(func, 0); @@ -988,10 +1008,10 @@ fn expand_f32_division( let den_inf_eq = b.equal_constant(func, denominator_magnitude, 0x7f80_0000); let denominator_infinity = bool_as_u32(func, b, den_inf_eq); let hh13 = b.constant(func, 0x7f80_0000); - let num_nan_gt = b.compare(func, CmpOp::Gt, numerator_magnitude, hh13); + let num_nan_gt = b.compare(func, CmpOp::Ugt, numerator_magnitude, hh13); let numerator_nan = bool_as_u32(func, b, num_nan_gt); let hh14 = b.constant(func, 0x7f80_0000); - let den_nan_gt = b.compare(func, CmpOp::Gt, denominator_magnitude, hh14); + let den_nan_gt = b.compare(func, CmpOp::Ugt, denominator_magnitude, hh14); let denominator_nan = bool_as_u32(func, b, den_nan_gt); let di_set = flag_set(func, b, denominator_infinity); let hh15 = b.constant(func, 0); @@ -1105,10 +1125,10 @@ pub fn expand_f32_div(func: &mut Function) -> bool { // --- Mulh -------------------------------------------------------------------- fn is_mulh(func: &Function, inst: Inst) -> bool { - matches!(func.opcode(inst), Opcode::Arith(a) if a.op == BinOp::Mulh) + matches!(func.opcode(inst), Opcode::Arith(a) if matches!(a.op, BinOp::UMulh | BinOp::SMulh)) } -/// Expand the high half of a full-width product into half-width limbs. +/// Expand the high half of a full-width product (`UMulh` and `SMulh`) into half-width limbs. /// Returns whether anything changed. pub fn expand_mulh(func: &mut Function) -> bool { let mut changed = false; @@ -1139,7 +1159,7 @@ pub fn expand_mulh(func: &mut Function) -> bool { continue; }; let ty = func.value_type(result); - let high = emit_mulh_limbs(func, &mut out, a.lhs, a.rhs, ty); + let high = emit_mulh_limbs(func, &mut out, a.lhs, a.rhs, ty, a.op == BinOp::SMulh); func.replace_all_uses(result, high); } func.set_block_insts(block, &out); @@ -1147,20 +1167,21 @@ pub fn expand_mulh(func: &mut Function) -> bool { changed } -/// Emit the limb sequence for `high half of (lhs * rhs)` at type `ty`, -/// appending each instruction to `out`, and return the high-half value. +/// Emit the limb sequence for the high half of `lhs * rhs` at type `ty`, read as unsigned +/// or (`signed`) as two's complement signed, appending each instruction to `out`, and return +/// the high-half value. fn emit_mulh_limbs( func: &mut Function, out: &mut Vec, lhs: Value, rhs: Value, ty: Type, + signed: bool, ) -> Value { - let info = match func.types.type_kind(ty) { - TypeKind::Int(i) => *i, + let w = match func.types.type_kind(ty) { + TypeKind::Int(i) => i.bits, _ => unreachable!("mulh is integer-only"), }; - let w = info.bits; let h = (w / 2) as i64; let mask_h: i64 = (1i64 << (w / 2)) - 1; @@ -1198,13 +1219,15 @@ fn emit_mulh_limbs( let s0 = b.op(BinOp::Add, hihi, lohi_hi); let s1 = b.op(BinOp::Add, s0, hilo_hi); let unsigned_high = b.op(BinOp::Add, s1, cross_hi); - if !info.signed { + if !signed { return unsigned_high; } + // Signed correction: subtract each operand where the other is negative (an arithmetic + // shift by `w - 1` is all ones for a negative operand, zero otherwise). let wm1 = b.konst((w - 1) as i64); - let amask = b.op(BinOp::Shr, lhs, wm1); - let bmask = b.op(BinOp::Shr, rhs, wm1); + let amask = b.op(BinOp::Sar, lhs, wm1); + let bmask = b.op(BinOp::Sar, rhs, wm1); let ca = b.op(BinOp::BitAnd, amask, rhs); let cb = b.op(BinOp::BitAnd, bmask, lhs); let c0 = b.op(BinOp::Sub, unsigned_high, ca); @@ -1240,7 +1263,11 @@ struct Shape { b_elem: Type, acc: Type, elem_bytes: i64, - convert: bool, + /// How a loaded element of A / B widens to the accumulator type; `None` when it already + /// has it. An int8 element is loaded as the raw byte (zero-extended, signless), so its + /// signedness is the choice of `Sext` or `Zext` here. + a_widen: Option, + b_widen: Option, acc_is_float: bool, } @@ -1249,14 +1276,15 @@ const C_ELEM_BYTES: i64 = 4; fn shape_of(func: &mut Function, mm: &MatMul) -> Shape { let f32_t = func.types.intern(TypeKind::Float(FloatKind::F32)); - let i32_t = func.types.intern(int_ty_kind(32, true)); + let i32_t = func.types.intern(int_ty_kind(32)); match mm.dtype { MatMulType::Fp32 => Shape { a_elem: f32_t, b_elem: f32_t, acc: f32_t, elem_bytes: 4, - convert: false, + a_widen: None, + b_widen: None, acc_is_float: true, }, MatMulType::Int8 | MatMulType::Uint8 => { @@ -1268,12 +1296,20 @@ fn shape_of(func: &mut Function, mm: &MatMul) -> Shape { Some(s) => (s.a_unsigned, s.b_unsigned), None => (default_unsigned, default_unsigned), }; + let widen = |unsigned: bool| { + Some(if unsigned { + ConvertKind::Zext + } else { + ConvertKind::Sext + }) + }; Shape { - a_elem: func.types.intern(int_ty_kind(8, !a_unsigned)), - b_elem: func.types.intern(int_ty_kind(8, !b_unsigned)), + a_elem: func.types.intern(int_ty_kind(8)), + b_elem: func.types.intern(int_ty_kind(8)), acc: i32_t, elem_bytes: 1, - convert: true, + a_widen: widen(a_unsigned), + b_widen: widen(b_unsigned), acc_is_float: false, } } @@ -1284,7 +1320,8 @@ fn shape_of(func: &mut Function, mm: &MatMul) -> Shape { b_elem: func.types.intern(TypeKind::Float(FloatKind::F16)), acc: func.types.intern(TypeKind::Float(FloatKind::F32)), elem_bytes: 2, - convert: true, + a_widen: Some(ConvertKind::FpResize), + b_widen: Some(ConvertKind::FpResize), acc_is_float: true, }, } @@ -1367,7 +1404,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { let ptr_t = func.types.ptr_global(); let bool_t = func.types.intern(TypeKind::Bool); - let i32_t = func.types.intern(int_ty_kind(32, true)); + let i32_t = func.types.intern(int_ty_kind(32)); // Snapshot the block's instruction list: `set_block_insts` clears the list // it would then read from, so the two halves must be copied out first. @@ -1412,7 +1449,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { nest.i_head, bool_t, Opcode::Icmp(super::function::Compare { - op: CmpOp::Lt, + op: CmpOp::Slt, lhs: i, rhs: m_bound, }), @@ -1436,7 +1473,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { nest.j_head, bool_t, Opcode::Icmp(super::function::Compare { - op: CmpOp::Lt, + op: CmpOp::Slt, lhs: j, rhs: n_bound, }), @@ -1452,14 +1489,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { // j-header also runs with `j == n`, where `c_elem` is one past the end of // the C row. let acc_init = if mm.accumulate { - func.append_inst( - nest.j_body, - shape.acc, - Opcode::Load(super::function::Load { - ptr: c_elem, - volatile: false, - }), - ) + func.append_load(nest.j_body, shape.acc, c_elem, MemFlags::new()) } else { acc_zero }; @@ -1474,7 +1504,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { nest.p_head, bool_t, Opcode::Icmp(super::function::Compare { - op: CmpOp::Lt, + op: CmpOp::Slt, lhs: p, rhs: k_bound, }), @@ -1486,39 +1516,15 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { super::function::EdgeDesc::bare(nest.j_latch), ); - let a_val = func.append_inst( - nest.p_body, - shape.a_elem, - Opcode::Load(super::function::Load { - ptr: a_elem, - volatile: false, - }), - ); - let b_val = func.append_inst( - nest.p_body, - shape.b_elem, - Opcode::Load(super::function::Load { - ptr: b_elem, - volatile: false, - }), - ); - let a_op = if shape.convert { - func.append_inst( - nest.p_body, - shape.acc, - Opcode::Convert(super::function::Convert { value: a_val }), - ) - } else { - a_val + let a_val = func.append_load(nest.p_body, shape.a_elem, a_elem, MemFlags::new()); + let b_val = func.append_load(nest.p_body, shape.b_elem, b_elem, MemFlags::new()); + let a_op = match shape.a_widen { + Some(kind) => func.append_convert(nest.p_body, shape.acc, kind, a_val), + None => a_val, }; - let b_op = if shape.convert { - func.append_inst( - nest.p_body, - shape.acc, - Opcode::Convert(super::function::Convert { value: b_val }), - ) - } else { - b_val + let b_op = match shape.b_widen { + Some(kind) => func.append_convert(nest.p_body, shape.acc, kind, b_val), + None => b_val, }; let product = func.append_inst( nest.p_body, diff --git a/src/ir/src/expand/tests/interp.rs b/src/ir/src/expand/tests/interp.rs new file mode 100644 index 0000000..6d3851a --- /dev/null +++ b/src/ir/src/expand/tests/interp.rs @@ -0,0 +1,326 @@ +//! A small reference interpreter for the IR the expansions produce, used to check their +//! numerics against the host implementations. Values are bit patterns, held zero-extended at +//! their type's width (the backend convention); `Sext`, `Sar`, `SDiv`, ... re-read the bits as +//! signed on demand. + +use alloc::vec::Vec; +use std::collections::HashMap; + +use crate::function::{ + BinOp, Block, CmpOp, ConvertKind, Function, Opcode, Terminator, UnaryOp, Value, +}; +use crate::types::{FloatKind, Type, TypeKind}; + +fn bits_of(func: &Function, ty: Type) -> u32 { + match func.types.type_kind(ty) { + TypeKind::Int(d) => u32::from(d.bits), + TypeKind::Bool => 1, + TypeKind::Ptr(_) => 64, + TypeKind::Float(FloatKind::F16) => 16, + TypeKind::Float(FloatKind::F32) => 32, + TypeKind::Float(FloatKind::F64) => 64, + other => panic!("interpreter: unsupported type {other:?}"), + } +} + +fn mask(bits: u32) -> u64 { + if bits >= 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + } +} + +fn sext(x: u64, bits: u32) -> i64 { + if bits >= 64 { + x as i64 + } else { + ((x << (64 - bits)) as i64) >> (64 - bits) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Fp { + F16, + F32, + F64, +} + +fn fp_kind(func: &Function, ty: Type) -> Option { + match func.types.type_kind(ty) { + TypeKind::Float(FloatKind::F16) => Some(Fp::F16), + TypeKind::Float(FloatKind::F32) => Some(Fp::F32), + TypeKind::Float(FloatKind::F64) => Some(Fp::F64), + TypeKind::Float(FloatKind::F128) => panic!("interpreter: f128"), + _ => None, + } +} + +fn f16_to_f64(bits: u64) -> f64 { + let sign = if bits & 0x8000 != 0 { -1.0 } else { 1.0 }; + let exp = ((bits >> 10) & 0x1f) as i32; + let frac = (bits & 0x3ff) as f64; + match exp { + 0 => sign * frac * 2f64.powi(-24), + 31 if frac == 0.0 => sign * f64::INFINITY, + 31 => f64::NAN, + _ => sign * (1.0 + frac / 1024.0) * 2f64.powi(exp - 15), + } +} + +/// The value of an IEEE half-precision bit pattern, exactly (every half is an f32). +pub fn f16_to_f32(bits: u16) -> f32 { + f16_to_f64(u64::from(bits)) as f32 +} + +fn to_f64(kind: Fp, bits: u64) -> f64 { + match kind { + Fp::F16 => f16_to_f64(bits), + Fp::F32 => f64::from(f32::from_bits(bits as u32)), + Fp::F64 => f64::from_bits(bits), + } +} + +fn from_f64(kind: Fp, v: f64) -> u64 { + match kind { + Fp::F16 => panic!("interpreter: narrowing to f16"), + Fp::F32 => u64::from((v as f32).to_bits()), + Fp::F64 => v.to_bits(), + } +} + +fn int_arith(op: BinOp, l: u64, r: u64, bits: u32) -> u64 { + let m = mask(bits); + let (sl, sr) = (sext(l, bits), sext(r, bits)); + let raw = match op { + BinOp::Add => l.wrapping_add(r), + BinOp::Sub => l.wrapping_sub(r), + BinOp::Mul => l.wrapping_mul(r), + BinOp::BitAnd => l & r, + BinOp::BitOr => l | r, + BinOp::BitXor => l ^ r, + BinOp::Shl => l.checked_shl(r as u32).unwrap_or(0), + BinOp::Shr => l.checked_shr(r as u32).unwrap_or(0), + BinOp::Sar => (sl >> r.min(63)) as u64, + BinOp::UDiv => l / r, + BinOp::URem => l % r, + BinOp::SDiv => sl.wrapping_div(sr) as u64, + BinOp::SRem => sl.wrapping_rem(sr) as u64, + BinOp::UMulh => ((u128::from(l) * u128::from(r)) >> bits) as u64, + BinOp::SMulh => ((i128::from(sl) * i128::from(sr)) >> bits) as u64, + BinOp::Div | BinOp::Rem => panic!("interpreter: {op:?} on an integer"), + }; + raw & m +} + +fn float_arith(op: BinOp, kind: Fp, l: u64, r: u64) -> u64 { + let (l, r) = (to_f64(kind, l), to_f64(kind, r)); + let v = match op { + BinOp::Add => l + r, + BinOp::Sub => l - r, + BinOp::Mul => l * r, + BinOp::Div => l / r, + BinOp::Rem => l % r, + other => panic!("interpreter: {other:?} on a float"), + }; + from_f64(kind, v) +} + +fn compare(op: CmpOp, kind: Option, l: u64, r: u64, bits: u32) -> bool { + if let Some(kind) = kind { + let (l, r) = (to_f64(kind, l), to_f64(kind, r)); + return match op { + CmpOp::Eq => l == r, + CmpOp::Ne => l != r, + CmpOp::Lt => l < r, + CmpOp::Le => l <= r, + CmpOp::Gt => l > r, + CmpOp::Ge => l >= r, + other => panic!("interpreter: {other:?} on a float"), + }; + } + let (sl, sr) = (sext(l, bits), sext(r, bits)); + match op { + CmpOp::Eq => l == r, + CmpOp::Ne => l != r, + CmpOp::Slt => sl < sr, + CmpOp::Sle => sl <= sr, + CmpOp::Sgt => sl > sr, + CmpOp::Sge => sl >= sr, + CmpOp::Ult => l < r, + CmpOp::Ule => l <= r, + CmpOp::Ugt => l > r, + CmpOp::Uge => l >= r, + other => panic!("interpreter: {other:?} on an integer"), + } +} + +/// Runs functions over a flat little-endian memory. +pub struct Interp { + pub mem: Vec, +} + +impl Interp { + pub fn new(mem: Vec) -> Self { + Self { mem } + } + + fn read(&self, addr: u64, bytes: u32) -> u64 { + (0..bytes as usize).fold(0u64, |acc, i| { + acc | u64::from(self.mem[addr as usize + i]) << (8 * i) + }) + } + + fn write(&mut self, addr: u64, bytes: u32, value: u64) { + for i in 0..bytes as usize { + self.mem[addr as usize + i] = (value >> (8 * i)) as u8; + } + } + + /// Executes `func` from its entry block with `args` bound to the entry parameters and + /// returns the returned values. + pub fn run(&mut self, func: &Function, args: &[u64]) -> Vec { + let mut vals: HashMap = HashMap::new(); + let mut block = Block::from_u32(0); + let entry_params = func.block_params(block).to_vec(); + assert_eq!(entry_params.len(), args.len()); + for (p, a) in entry_params.iter().zip(args) { + vals.insert(*p, *a & mask(bits_of(func, func.value_type(*p)))); + } + let mut steps = 0u64; + 'blocks: loop { + for &inst in func.block_insts(block) { + steps += 1; + assert!(steps < 5_000_000, "interpreter: runaway loop"); + let result = func.inst_result(inst); + let result_ty = result.map(|r| func.value_type(r)); + let out: Option = match func.opcode(inst) { + Opcode::Iconst(c) => Some(c as u64 & mask(bits_of(func, result_ty.unwrap()))), + Opcode::Fconst(f) => Some(match fp_kind(func, result_ty.unwrap()) { + Some(Fp::F32) => u64::from((f as f32).to_bits()), + Some(Fp::F64) => f.to_bits(), + other => panic!("interpreter: fconst {other:?}"), + }), + Opcode::Arith(a) => { + let ty = func.value_type(a.lhs); + let (l, r) = (vals[&a.lhs], vals[&a.rhs]); + Some(match fp_kind(func, ty) { + Some(kind) => float_arith(a.op, kind, l, r), + None => int_arith(a.op, l, r, bits_of(func, ty)), + }) + } + Opcode::ArithImm(a) => { + let ty = func.value_type(a.lhs); + let bits = bits_of(func, ty); + Some(int_arith( + a.op, + vals[&a.lhs], + a.imm as u64 & mask(bits), + bits, + )) + } + Opcode::Icmp(c) => { + let ty = func.value_type(c.lhs); + Some(u64::from(compare( + c.op, + fp_kind(func, ty), + vals[&c.lhs], + vals[&c.rhs], + bits_of(func, ty), + ))) + } + Opcode::Select(s) => Some(if vals[&s.cond] != 0 { + vals[&s.then] + } else { + vals[&s.else_] + }), + Opcode::Convert(cv) => { + let (src_ty, dst_ty) = (func.value_type(cv.value), result_ty.unwrap()); + let (sb, db) = (bits_of(func, src_ty), bits_of(func, dst_ty)); + let v = vals[&cv.value]; + Some(match cv.kind { + ConvertKind::Trunc => { + assert!(db < sb); + v & mask(db) + } + ConvertKind::Zext => { + assert!(db > sb); + v + } + ConvertKind::Sext => { + assert!(db > sb); + sext(v, sb) as u64 & mask(db) + } + ConvertKind::SiToFp => { + from_f64(fp_kind(func, dst_ty).unwrap(), sext(v, sb) as f64) + } + ConvertKind::UiToFp => { + from_f64(fp_kind(func, dst_ty).unwrap(), v as f64) + } + ConvertKind::FpToSi => { + let f = to_f64(fp_kind(func, src_ty).unwrap(), v); + let (lo, hi) = if db >= 64 { + (i64::MIN, i64::MAX) + } else { + (-(1i64 << (db - 1)), (1i64 << (db - 1)) - 1) + }; + (f as i64).clamp(lo, hi) as u64 & mask(db) + } + ConvertKind::FpToUi => { + let f = to_f64(fp_kind(func, src_ty).unwrap(), v); + (f as u64).min(mask(db)) + } + ConvertKind::FpResize => { + let f = to_f64(fp_kind(func, src_ty).unwrap(), v); + from_f64(fp_kind(func, dst_ty).unwrap(), f) + } + }) + } + Opcode::Unary(u) => { + assert_eq!(u.op, UnaryOp::Reinterpret); + let bits = bits_of(func, result_ty.unwrap()); + assert_eq!(bits, bits_of(func, func.value_type(u.value))); + Some(vals[&u.value] & mask(bits)) + } + Opcode::Load(l) => { + let bytes = bits_of(func, result_ty.unwrap()) / 8; + Some(self.read(vals[&l.ptr], bytes)) + } + Opcode::Store(st) => { + let bytes = bits_of(func, func.value_type(st.value)) / 8; + self.write(vals[&st.ptr], bytes, vals[&st.value]); + None + } + Opcode::If(cf) => { + let jump = if vals[&cf.cond] != 0 { + cf.then + } else { + cf.else_ + }; + let moved: Vec = + func.block_args(jump).iter().map(|a| vals[a]).collect(); + for (p, v) in func.block_params(jump.target).iter().zip(moved) { + vals.insert(*p, v); + } + block = jump.target; + continue 'blocks; + } + other => panic!("interpreter: unsupported opcode {other:?}"), + }; + if let (Some(r), Some(v)) = (result, out) { + vals.insert(r, v); + } + } + match func.terminator(block).expect("terminated block") { + Terminator::Ret(r) => return r.slice().iter().map(|v| vals[v]).collect(), + Terminator::Jump(j) => { + let moved: Vec = func.block_args(j).iter().map(|a| vals[a]).collect(); + for (p, v) in func.block_params(j.target).iter().zip(moved) { + vals.insert(*p, v); + } + block = j.target; + } + } + } + } +} diff --git a/src/ir/src/expand/tests/mod.rs b/src/ir/src/expand/tests/mod.rs index 3edc19d..2416e62 100644 --- a/src/ir/src/expand/tests/mod.rs +++ b/src/ir/src/expand/tests/mod.rs @@ -1,2 +1,3 @@ +mod interp; mod tests_matmul; mod tests_scalar; diff --git a/src/ir/src/expand/tests/tests_matmul.rs b/src/ir/src/expand/tests/tests_matmul.rs index a4ce189..dd47a10 100644 --- a/src/ir/src/expand/tests/tests_matmul.rs +++ b/src/ir/src/expand/tests/tests_matmul.rs @@ -2,13 +2,15 @@ use alloc::string::ToString; +use super::interp::{Interp, f16_to_f32}; use crate::attribute::Attribute; use crate::expand::expand_matmul; use crate::function::{ - Arith, AttrTarget, Function, MatMulQuant, MatMulQuantOut, MatMulScale, MatMulType, Opcode, Ret, - Terminator, + Arith, AttrTarget, ConvertKind, Function, InputSigns, MatMulQuant, MatMulQuantOut, MatMulScale, + MatMulType, Opcode, Ret, Terminator, }; use crate::types::{FloatKind, Type, TypeKind}; +use crate::verify::{Profile, verify}; use alloc::{vec, vec::Vec}; fn f32_ty(func: &mut Function) -> Type { @@ -173,3 +175,213 @@ fn expand_matmul_skips_block_keyed_attributes() { ); assert!(!expand_matmul(&mut func)); } + +/// Builds `fn() { matmul c, a, b [m x n x k] }` over the flat memory layout A, B, C (in that +/// order, `a_len` and `b_len` bytes long), expands it, checks it still verifies and runs it +/// on `mem`, returning the final memory. +#[allow(clippy::too_many_arguments)] +fn run_matmul( + dtype: MatMulType, + (m, n, k): (u16, u16, u16), + signs: Option, + accumulate: bool, + (a_len, b_len): (usize, usize), + mem: Vec, +) -> (Function, Vec) { + let mut func = Function::new(); + let ptr_t = func.types.ptr_global(); + let block = func.append_block(); + let a = func.append_inst(block, ptr_t, Opcode::Iconst(0)); + let b = func.append_inst(block, ptr_t, Opcode::Iconst(a_len as i64)); + let c = func.append_inst(block, ptr_t, Opcode::Iconst((a_len + b_len) as i64)); + func.append_matmul_embedded(block, a, b, c, m, n, k, dtype, accumulate, signs); + func.set_terminator(block, Terminator::Ret(Ret::none())); + assert!(expand_matmul(&mut func)); + let diags = verify(&func, Profile::High); + assert!(diags.ok(), "{:?}", diags.items()); + let mut interp = Interp::new(mem); + interp.run(&func, &[]); + (func, interp.mem) +} + +fn pseudo_random_bytes(len: usize, mut seed: u64) -> Vec { + (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed >> 24) as u8 + }) + .collect() +} + +/// Runs an int8 matmul and compares C with the plain host computation, reading each operand +/// as unsigned or signed as `a_unsigned` / `b_unsigned` say. +fn check_int8( + dtype: MatMulType, + signs: Option, + a_unsigned: bool, + b_unsigned: bool, + accumulate: bool, +) { + let (m, n, k) = (3usize, 4usize, 5usize); + let a = pseudo_random_bytes(m * k, 0x9e37_79b9_7f4a_7c15); + let b = pseudo_random_bytes(k * n, 0x1234_5678_9abc_def1); + let c_init = pseudo_random_bytes(m * n * 4, 0x0fed_cba9_8765_4321); + let mut mem = a.clone(); + mem.extend_from_slice(&b); + mem.extend_from_slice(&c_init); + let (func, out) = run_matmul( + dtype, + (m as u16, n as u16, k as u16), + signs, + accumulate, + (a.len(), b.len()), + mem, + ); + let widen = |x: u8, unsigned: bool| { + if unsigned { + i32::from(x) + } else { + i32::from(x as i8) + } + }; + for i in 0..m { + for j in 0..n { + let at = 4 * (i * n + j); + let mut want = if accumulate { + i32::from_le_bytes(c_init[at..at + 4].try_into().unwrap()) + } else { + 0 + }; + for p in 0..k { + want = want.wrapping_add( + widen(a[i * k + p], a_unsigned) * widen(b[p * n + j], b_unsigned), + ); + } + let base = a.len() + b.len() + at; + let got = i32::from_le_bytes(out[base..base + 4].try_into().unwrap()); + assert_eq!( + got, want, + "{dtype:?} {signs:?} acc={accumulate} c[{i}][{j}]" + ); + } + } + // The widening is the explicit part of the expansion: one `Sext` or `Zext` per operand. + let widens = |kind: ConvertKind| { + count( + &func, + |op| matches!(op, Opcode::Convert(c) if c.kind == kind), + ) + }; + let (zexts, sexts) = (widens(ConvertKind::Zext), widens(ConvertKind::Sext)); + assert_eq!(zexts, usize::from(a_unsigned) + usize::from(b_unsigned)); + assert_eq!(sexts, usize::from(!a_unsigned) + usize::from(!b_unsigned)); +} + +#[test] +fn expand_matmul_int8_reads_both_operands_signed() { + check_int8(MatMulType::Int8, None, false, false, false); + check_int8(MatMulType::Int8, None, false, false, true); +} + +#[test] +fn expand_matmul_uint8_reads_both_operands_unsigned() { + check_int8(MatMulType::Uint8, None, true, true, false); + check_int8(MatMulType::Uint8, None, true, true, true); +} + +#[test] +fn expand_matmul_input_signs_override_the_dtype_per_operand() { + let mixed = Some(InputSigns { + a_unsigned: true, + b_unsigned: false, + }); + check_int8(MatMulType::Int8, mixed, true, false, false); + check_int8(MatMulType::Uint8, mixed, true, false, true); + let other = Some(InputSigns { + a_unsigned: false, + b_unsigned: true, + }); + check_int8(MatMulType::Uint8, other, false, true, false); + check_int8(MatMulType::Int8, other, false, true, true); +} + +#[test] +fn expand_matmul_fp16_widens_with_fp_resize() { + let (m, n, k) = (2usize, 3usize, 4usize); + // Finite halves, positive and negative: 1.0, -2.5, 0.375, 100.0, -0.0625, ... + let halves: [u16; 8] = [ + 0x3c00, 0xc100, 0x3600, 0x5640, 0xac00, 0x4200, 0xbd00, 0x0400, + ]; + let pick = |i: usize| halves[i % halves.len()]; + let a: Vec = (0..m * k).map(|i| pick(i * 3 + 1)).collect(); + let b: Vec = (0..k * n).map(|i| pick(i * 5 + 2)).collect(); + let mut mem: Vec = Vec::new(); + for h in a.iter().chain(&b) { + mem.extend_from_slice(&h.to_le_bytes()); + } + let (a_len, b_len) = (a.len() * 2, b.len() * 2); + mem.extend(core::iter::repeat_n(0u8, m * n * 4)); + let (func, out) = run_matmul( + MatMulType::Fp16, + (m as u16, n as u16, k as u16), + None, + false, + (a_len, b_len), + mem, + ); + for i in 0..m { + for j in 0..n { + let mut want = 0.0f32; + for p in 0..k { + want += f16_to_f32(a[i * k + p]) * f16_to_f32(b[p * n + j]); + } + let base = a_len + b_len + 4 * (i * n + j); + let got = f32::from_le_bytes(out[base..base + 4].try_into().unwrap()); + assert_eq!(got.to_bits(), want.to_bits(), "c[{i}][{j}]"); + } + } + assert_eq!( + count( + &func, + |op| matches!(op, Opcode::Convert(c) if c.kind == ConvertKind::FpResize) + ), + 2 + ); +} + +#[test] +fn expand_matmul_fp32_accumulates_in_k_order() { + let (m, n, k) = (2usize, 2usize, 3usize); + let a = [1.5f32, -2.0, 0.25, 3.0, 0.5, -1.0]; + let b = [2.0f32, 4.0, -1.0, 0.5, 8.0, 3.0]; + let mut mem: Vec = Vec::new(); + for x in a.iter().chain(&b) { + mem.extend_from_slice(&x.to_le_bytes()); + } + let c_init = [10.0f32, 20.0, 30.0, 40.0]; + for x in &c_init { + mem.extend_from_slice(&x.to_le_bytes()); + } + let (func, out) = run_matmul( + MatMulType::Fp32, + (m as u16, n as u16, k as u16), + None, + true, + (a.len() * 4, b.len() * 4), + mem, + ); + for i in 0..m { + for j in 0..n { + let mut want = c_init[i * n + j]; + for p in 0..k { + want += a[i * k + p] * b[p * n + j]; + } + let base = (a.len() + b.len()) * 4 + 4 * (i * n + j); + let got = f32::from_le_bytes(out[base..base + 4].try_into().unwrap()); + assert_eq!(got.to_bits(), want.to_bits(), "c[{i}][{j}]"); + } + } + assert_eq!(count(&func, |op| matches!(op, Opcode::Convert(_))), 0); +} diff --git a/src/ir/src/expand/tests/tests_scalar.rs b/src/ir/src/expand/tests/tests_scalar.rs index 2e74c27..5c8cf8c 100644 --- a/src/ir/src/expand/tests/tests_scalar.rs +++ b/src/ir/src/expand/tests/tests_scalar.rs @@ -4,34 +4,40 @@ use alloc::string::ToString; use alloc::vec::Vec; use super::super::*; +use super::interp::Interp; use crate::attribute::{AttrValue, Attribute, Custom}; -use crate::expand::{expand_low_float, expand_mulh, expand_nv_fp4}; +use crate::expand::{expand_f32_div, expand_low_float, expand_mulh, expand_nv_fp4}; use crate::function::AttrTarget; use crate::function::Function; use crate::function::{Arith, Ret, Terminator, Unary, UnaryOp}; +use crate::low_float::{self, Format}; +use crate::nvfp4::{self, ScaleApplication}; use crate::types::{IntDesc, Type, TypeKind}; +use crate::verify::{Profile, verify}; -fn unsigned_int(func: &mut Function, bits: u16) -> Type { - func.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits, - })) +fn int_ty(func: &mut Function, bits: u16) -> Type { + func.types.intern(TypeKind::Int(IntDesc { bits })) +} + +fn assert_verifies(func: &Function) { + let diags = verify(func, Profile::High); + assert!(diags.ok(), "{:?}", diags.items()); } fn f32_ty(func: &mut Function) -> Type { func.types.intern(TypeKind::Float(FloatKind::F32)) } -fn u8_ty(func: &mut Function) -> Type { - unsigned_int(func, 8) +fn i8_ty(func: &mut Function) -> Type { + int_ty(func, 8) } -fn u16_ty(func: &mut Function) -> Type { - unsigned_int(func, 16) +fn i16_ty(func: &mut Function) -> Type { + int_ty(func, 16) } -fn u32_ty(func: &mut Function) -> Type { - unsigned_int(func, 32) +fn i32_ty(func: &mut Function) -> Type { + int_ty(func, 32) } fn custom_attr(key: &str) -> Attribute { @@ -48,8 +54,8 @@ fn custom_attr(key: &str) -> Attribute { fn expand_low_float_no_op_without_trigger() { let mut func = Function::new(); let block = func.append_block(); - let u32t = u32_ty(&mut func); - let v = func.append_inst(block, u32t, Opcode::Iconst(41)); + let i32t = i32_ty(&mut func); + let v = func.append_inst(block, i32t, Opcode::Iconst(41)); let inst = func.defining_inst(v).unwrap(); func.add_attr(AttrTarget::Value(v), custom_attr("unchanged")); func.set_terminator(block, Terminator::Ret(Ret::one(v))); @@ -64,7 +70,7 @@ fn expand_low_float_no_op_without_trigger() { #[test] fn expand_low_float_rewrites_and_migrates_attrs() { let mut func = Function::new(); - let payload_t = u16_ty(&mut func); + let payload_t = i16_ty(&mut func); let f32t = f32_ty(&mut func); let block = func.append_block(); let payload = func.append_block_param(block, payload_t); @@ -125,12 +131,12 @@ fn expand_low_float_rewrites_and_migrates_attrs() { fn expand_low_float_encode_fp8_moves_attrs_to_f32_boundary() { let mut func = Function::new(); let f32t = f32_ty(&mut func); - let u8t = u8_ty(&mut func); + let i8t = i8_ty(&mut func); let block = func.append_block(); let source = func.append_block_param(block, f32t); let encoded = func.append_inst( block, - u8t, + i8t, Opcode::EncodeLowFloat(crate::function::LowFloatConvert { value: source, format: crate::low_float::Format::F8E5M2, @@ -171,11 +177,11 @@ fn expand_low_float_encode_fp8_moves_attrs_to_f32_boundary() { // --- NVFP4 ------------------------------------------------------------------- fn nvfp4_fixture(func: &mut Function, dequant: bool) -> (Block, Value) { - let u8t = u8_ty(func); + let i8t = i8_ty(func); let f32t = f32_ty(func); let block = func.append_block(); - let payload = func.append_block_param(block, u8t); - let block_scale = func.append_block_param(block, u8t); + let payload = func.append_block_param(block, i8t); + let block_scale = func.append_block_param(block, i8t); let value = func.append_block_param(block, f32t); let global_scale = func.append_block_param(block, f32t); let conv = crate::function::NvFp4Convert { @@ -188,7 +194,7 @@ fn nvfp4_fixture(func: &mut Function, dequant: bool) -> (Block, Value) { let result = if dequant { func.append_inst(block, f32t, Opcode::DequantizeNvfp4(conv)) } else { - func.append_inst(block, u8t, Opcode::QuantizeNvfp4(conv)) + func.append_inst(block, i8t, Opcode::QuantizeNvfp4(conv)) }; func.set_terminator(block, Terminator::Ret(Ret::one(result))); (block, result) @@ -198,8 +204,8 @@ fn nvfp4_fixture(func: &mut Function, dequant: bool) -> (Block, Value) { fn expand_nv_fp4_no_op_without_trigger() { let mut func = Function::new(); let block = func.append_block(); - let u32t = u32_ty(&mut func); - let v = func.append_inst(block, u32t, Opcode::Iconst(43)); + let i32t = i32_ty(&mut func); + let v = func.append_inst(block, i32t, Opcode::Iconst(43)); func.set_terminator(block, Terminator::Ret(Ret::one(v))); assert!(!expand_nv_fp4(&mut func)); assert!(matches!( @@ -318,45 +324,40 @@ fn expand_nv_fp4_migrates_attrs_to_semantic_boundary() { // --- Mulh -------------------------------------------------------------------- -fn mulh_const_fixture(func: &mut Function, bits: u16, signed: bool) -> (Block, Inst) { - let ty = func.types.intern(TypeKind::Int(IntDesc { signed, bits })); +fn mulh_const_fixture(func: &mut Function, bits: u16, op: BinOp) -> (Block, Inst) { + let ty = int_ty(func, bits); let block = func.append_block(); let a = func.append_inst(block, ty, Opcode::Iconst(0x1234_5678_9abcu64 as i64)); let b = func.append_inst(block, ty, Opcode::Iconst(0xfeed_beefu64 as i64)); - let r = func.append_inst( - block, - ty, - Opcode::Arith(Arith { - op: BinOp::Mulh, - lhs: a, - rhs: b, - }), - ); + let r = func.append_inst(block, ty, Opcode::Arith(Arith { op, lhs: a, rhs: b })); func.set_terminator(block, Terminator::Ret(Ret::one(r))); (block, func.defining_inst(r).unwrap()) } #[test] fn expand_mulh_no_mulh_survives_and_idempotent() { - let mut func = Function::new(); - let (block, _) = mulh_const_fixture(&mut func, 64, true); - assert!(expand_mulh(&mut func)); - assert!(!expand_mulh(&mut func)); - for inst in func.block_insts(block).to_vec() { - assert!(!matches!( - func.opcode(inst), - Opcode::Arith(Arith { - op: BinOp::Mulh, - .. - }) - )); + for op in [BinOp::UMulh, BinOp::SMulh] { + let mut func = Function::new(); + let (block, _) = mulh_const_fixture(&mut func, 64, op); + assert!(expand_mulh(&mut func)); + assert!(!expand_mulh(&mut func)); + for inst in func.block_insts(block).to_vec() { + assert!(!matches!( + func.opcode(inst), + Opcode::Arith(Arith { + op: BinOp::UMulh | BinOp::SMulh, + .. + }) + )); + } + assert_verifies(&func); } } #[test] fn expand_mulh_false_without_trigger() { let mut func = Function::new(); - let ty = u32_ty(&mut func); + let ty = i32_ty(&mut func); let block = func.append_block(); let a = func.append_inst(block, ty, Opcode::Iconst(3)); let b = func.append_inst(block, ty, Opcode::Iconst(7)); @@ -373,124 +374,351 @@ fn expand_mulh_false_without_trigger() { assert!(!expand_mulh(&mut func)); } -#[test] -fn expand_mulh_limb_math_matches_oracle_signed() { +/// The value `fn() { ret }` returns, computed by the expanded limb math. +fn expanded_mulh(bits: u16, op: BinOp, lhs: u64, rhs: u64) -> u64 { let mut func = Function::new(); - let (block, _) = mulh_const_fixture(&mut func, 64, true); + let ty = int_ty(&mut func, bits); + let block = func.append_block(); + let a = func.append_inst(block, ty, Opcode::Iconst(lhs as i64)); + let b = func.append_inst(block, ty, Opcode::Iconst(rhs as i64)); + let r = func.append_inst(block, ty, Opcode::Arith(Arith { op, lhs: a, rhs: b })); + func.set_terminator(block, Terminator::Ret(Ret::one(r))); assert!(expand_mulh(&mut func)); - // Evaluate constant dataflow at declared width: high = (a*b as i128) >> 64 - let ret_v = match func.terminator(block) { - Some(Terminator::Ret(r)) => r.values[0], - _ => panic!("ret"), - }; - let got = eval_const(&func, ret_v); + assert_verifies(&func); + Interp::new(Vec::new()).run(&func, &[])[0] +} + +#[test] +fn expand_mulh_limb_math_matches_oracle_signed() { let a = 0x1234_5678_9abcu64 as i64; let b = 0xfeed_beefu64 as i64; let oracle = ((a as i128) * (b as i128)) >> 64; - assert_eq!(got, oracle as i64); + assert_eq!( + expanded_mulh(64, BinOp::SMulh, a as u64, b as u64), + oracle as u64 + ); } #[test] fn expand_mulh_limb_math_matches_oracle_unsigned() { - let mut func = Function::new(); - let (block, _) = mulh_const_fixture(&mut func, 64, false); - assert!(expand_mulh(&mut func)); - let ret_v = match func.terminator(block) { - Some(Terminator::Ret(r)) => r.values[0], - _ => panic!("ret"), - }; - let got = eval_const(&func, ret_v) as u64; let a = 0x1234_5678_9abcu64; let b = 0xfeed_beefu64; let oracle = ((a as u128) * (b as u128)) >> 64; - assert_eq!(got, oracle as u64); + assert_eq!(expanded_mulh(64, BinOp::UMulh, a, b), oracle as u64); } #[test] fn expand_mulh_works_at_32_bit_width() { - let mut func = Function::new(); - let ty = signed_int32(&mut func); - let block = func.append_block(); - let a = func.append_inst(block, ty, Opcode::Iconst(100_000)); - let b = func.append_inst(block, ty, Opcode::Iconst(100_000)); - let r = func.append_inst( - block, - ty, - Opcode::Arith(Arith { - op: BinOp::Mulh, - lhs: a, - rhs: b, - }), - ); - func.set_terminator(block, Terminator::Ret(Ret::one(r))); - assert!(expand_mulh(&mut func)); - let ret_v = match func.terminator(block) { - Some(Terminator::Ret(r)) => r.values[0], - _ => panic!("ret"), - }; - let got = eval_const(&func, ret_v); let oracle = ((100_000i128) * (100_000i128)) >> 32; - assert_eq!(got, oracle as i64); -} - -fn signed_int32(func: &mut Function) -> Type { - func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })) + assert_eq!( + expanded_mulh(32, BinOp::SMulh, 100_000, 100_000), + oracle as u64 + ); + assert_eq!( + expanded_mulh(32, BinOp::UMulh, 100_000, 100_000), + oracle as u64 + ); } -/// Evaluate a constants-only dataflow at the result type's width. -fn eval_const(func: &Function, v: Value) -> i64 { - let info = match func.types.type_kind(func.value_type(v)) { - TypeKind::Int(i) => *i, - _ => panic!("int expected"), - }; - let wrap = |x: i64| -> i64 { - let bits = info.bits; - if bits >= 64 { - return x; - } - let mask: u64 = if bits >= 64 { !0 } else { (1u64 << bits) - 1 }; - let low = (x as u64) & mask; - if info.signed { - let sign = 1u64 << (bits - 1); - if low & sign != 0 { - (low | !mask) as i64 +#[test] +fn expand_mulh_matches_the_oracle_at_every_width_for_both_signs() { + let samples: [u64; 9] = [ + 0, + 1, + 2, + 0x7f, + 0x80, + 0xff, + 0x8000_0001, + 0x1234_5678_9abc_def0, + u64::MAX, + ]; + for bits in [8u16, 16, 32, 64] { + let mask = if bits == 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + }; + let signed = |x: u64| { + if bits == 64 { + x as i64 } else { - low as i64 + ((x << (64 - bits)) as i64) >> (64 - bits) + } + }; + for &a in &samples { + for &b in &samples { + let (a, b) = (a & mask, b & mask); + let unsigned_want = ((u128::from(a) * u128::from(b)) >> bits) as u64 & mask; + let signed_want = + ((i128::from(signed(a)) * i128::from(signed(b))) >> bits) as u64 & mask; + assert_eq!( + expanded_mulh(bits, BinOp::UMulh, a, b), + unsigned_want, + "umulh i{bits} {a:#x} {b:#x}" + ); + assert_eq!( + expanded_mulh(bits, BinOp::SMulh, a, b), + signed_want, + "smulh i{bits} {a:#x} {b:#x}" + ); } + } + } +} + +// --- Numerics of the scalar expansions ---------------------------------------- + +/// Equal bit patterns, or both NaN (the expansions need not preserve a NaN payload). +fn same_f32(got: u32, want: u32) -> bool { + got == want || (f32::from_bits(got).is_nan() && f32::from_bits(want).is_nan()) +} + +fn sample_floats() -> Vec { + let mut v = alloc::vec![ + 0.0, + -0.0, + 1.0, + -1.0, + 1.5, + -2.5, + 0.1, + -0.1, + 3.7, + 1e-3, + 1e-6, + 1e-10, + 0.001_953_125, + 0.000_976_562_5, + 6.0, + 5.0, + 0.75, + 0.25, + 448.0, + 449.0, + 464.0, + 57344.0, + 61440.0, + 65504.0, + 1e10, + -1e10, + f32::MAX, + f32::MIN_POSITIVE, + f32::from_bits(1), + f32::from_bits(0x007f_ffff), + f32::INFINITY, + f32::NEG_INFINITY, + f32::NAN, + ]; + let mut s = 0x1234_5678_9abc_def0u64; + for _ in 0..1500 { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + v.push(f32::from_bits(s as u32)); + // Values near 1 exercise the rounding paths of the narrow formats. + v.push(f32::from_bits(0x3f00_0000 + (s >> 40) as u32 % 0x0100_0000)); + } + v +} + +#[test] +fn expanded_low_float_decode_matches_the_host_decoder() { + for format in [Format::Bf16, Format::F8E4M3, Format::F8E5M2] { + let mut func = Function::new(); + let payload_t = int_ty(&mut func, format.payload_bits()); + let f32t = f32_ty(&mut func); + let block = func.append_block(); + let payload = func.append_block_param(block, payload_t); + let decoded = func.append_inst( + block, + f32t, + Opcode::DecodeLowFloat(crate::function::LowFloatConvert { + value: payload, + format, + }), + ); + func.set_terminator(block, Terminator::Ret(Ret::one(decoded))); + assert!(expand_low_float(&mut func)); + assert_verifies(&func); + let payloads: Vec = if format == Format::Bf16 { + alloc::vec![ + 0, 1, 0x3f80, 0x8000, 0x7f80, 0xff80, 0x7fc0, 0xc2f7, 0x0001, 0x007f, 0x8001, + 0xffff + ] } else { - low as i64 + (0..=255).collect() + }; + for p in payloads { + let got = Interp::new(Vec::new()).run(&func, &[u64::from(p)])[0] as u32; + let want = low_float::decode(format, p).unwrap().to_bits(); + assert!( + same_f32(got, want), + "{format:?} {p:#x}: {got:#x} != {want:#x}" + ); + } + } +} + +#[test] +fn expanded_low_float_encode_matches_the_host_encoder() { + for format in [Format::Bf16, Format::F8E4M3, Format::F8E5M2] { + let mut func = Function::new(); + let payload_t = int_ty(&mut func, format.payload_bits()); + let f32t = f32_ty(&mut func); + let block = func.append_block(); + let value = func.append_block_param(block, f32t); + let encoded = func.append_inst( + block, + payload_t, + Opcode::EncodeLowFloat(crate::function::LowFloatConvert { value, format }), + ); + func.set_terminator(block, Terminator::Ret(Ret::one(encoded))); + assert!(expand_low_float(&mut func)); + assert_verifies(&func); + for v in sample_floats() { + let got = Interp::new(Vec::new()).run(&func, &[u64::from(v.to_bits())])[0]; + let want = u64::from(low_float::encode(format, v)); + assert_eq!(got, want, "{format:?} {v:e} ({:#x})", v.to_bits()); } + } +} + +const APPLICATIONS: [ScaleApplication; 2] = [ScaleApplication::Multiply, ScaleApplication::Divide]; + +fn nvfp4_function( + dequant: bool, + block_application: ScaleApplication, + global_application: ScaleApplication, +) -> Function { + let mut func = Function::new(); + let i8t = i8_ty(&mut func); + let f32t = f32_ty(&mut func); + let block = func.append_block(); + let payload = func.append_block_param(block, i8t); + let value = func.append_block_param(block, f32t); + let block_scale = func.append_block_param(block, i8t); + let global_scale = func.append_block_param(block, f32t); + let conv = crate::function::NvFp4Convert { + value: if dequant { payload } else { value }, + block_scale, + global_scale, + block_application, + global_application, }; - let inst = func.defining_inst(v).unwrap(); - match func.opcode(inst) { - Opcode::Iconst(c) => wrap(c), - Opcode::Arith(a) => { - let lhs = eval_const(func, a.lhs); - let rhs = eval_const(func, a.rhs); - let raw: i64 = match a.op { - BinOp::Add => lhs.wrapping_add(rhs), - BinOp::Sub => lhs.wrapping_sub(rhs), - BinOp::Mul => lhs.wrapping_mul(rhs), - BinOp::BitAnd => lhs & rhs, - BinOp::BitOr => lhs | rhs, - BinOp::Shr => lhs.wrapping_shr(rhs as u32), - BinOp::Shl => lhs.wrapping_shl(rhs as u32), - _ => panic!("unexpected op"), - }; - wrap(raw) + let result = if dequant { + func.append_inst(block, f32t, Opcode::DequantizeNvfp4(conv)) + } else { + func.append_inst(block, i8t, Opcode::QuantizeNvfp4(conv)) + }; + func.set_terminator(block, Terminator::Ret(Ret::one(result))); + func +} + +#[test] +fn expanded_nvfp4_dequantize_matches_the_host() { + for block_application in APPLICATIONS { + for global_application in APPLICATIONS { + let mut func = nvfp4_function(true, block_application, global_application); + assert!(expand_nv_fp4(&mut func)); + assert_verifies(&func); + for payload in 0..16u8 { + for scale in [0x00u8, 0x01, 0x08, 0x38, 0x40, 0x7e, 0x7f, 0xb8, 0xff] { + for global in [1.0f32, 0.5, 3.0, -2.0] { + let got = Interp::new(Vec::new()).run( + &func, + &[ + u64::from(payload), + 0, + u64::from(scale), + u64::from(global.to_bits()), + ], + )[0] as u32; + let want = nvfp4::dequantize( + payload, + scale, + global, + block_application, + global_application, + ) + .to_bits(); + assert!( + same_f32(got, want), + "{payload:#x} {scale:#x} {global}: {got:#x} != {want:#x}" + ); + } + } + } } - Opcode::Select(sel) => { - let cond = eval_const(func, sel.cond); - if cond != 0 { - eval_const(func, sel.then) - } else { - eval_const(func, sel.else_) + } +} + +#[test] +fn expanded_nvfp4_quantize_matches_the_host() { + for block_application in APPLICATIONS { + for global_application in APPLICATIONS { + let mut func = nvfp4_function(false, block_application, global_application); + assert!(expand_nv_fp4(&mut func)); + assert_verifies(&func); + for value in sample_floats().into_iter().take(400) { + for scale in [0x38u8, 0x40, 0x7e] { + let got = Interp::new(Vec::new()).run( + &func, + &[ + 0, + u64::from(value.to_bits()), + u64::from(scale), + u64::from(2.0f32.to_bits()), + ], + )[0]; + let want = u64::from(nvfp4::quantize( + value, + scale, + 2.0, + block_application, + global_application, + )); + assert_eq!(got, want, "{value:e} {scale:#x}"); + } } } - Opcode::Convert(cv) => eval_const(func, cv.value), - _ => panic!("unexpected opcode"), + } +} + +#[test] +fn expanded_f32_division_matches_the_hardware() { + let mut func = Function::new(); + let f32t = f32_ty(&mut func); + let block = func.append_block(); + let a = func.append_block_param(block, f32t); + let b = func.append_block_param(block, f32t); + let q = func.append_inst( + block, + f32t, + Opcode::Arith(Arith { + op: BinOp::Div, + lhs: a, + rhs: b, + }), + ); + func.set_terminator(block, Terminator::Ret(Ret::one(q))); + assert!(expand_f32_div(&mut func)); + assert!(!expand_f32_div(&mut func)); + assert_verifies(&func); + for inst in func.block_insts(block).to_vec() { + assert!(!matches!( + func.opcode(inst), + Opcode::Arith(Arith { op: BinOp::Div, .. }) + )); + } + let samples = sample_floats(); + for (i, &x) in samples.iter().enumerate() { + for &y in samples.iter().skip(i % 7).step_by(23).take(60) { + let got = Interp::new(Vec::new()) + .run(&func, &[u64::from(x.to_bits()), u64::from(y.to_bits())])[0] + as u32; + let want = (x / y).to_bits(); + assert!(same_f32(got, want), "{x:e} / {y:e}: {got:#x} != {want:#x}"); + } } } diff --git a/src/ir/src/function.rs b/src/ir/src/function.rs index 23d48d3..456d659 100644 --- a/src/ir/src/function.rs +++ b/src/ir/src/function.rs @@ -3,7 +3,7 @@ extern crate alloc; use alloc::string::String; use alloc::vec::Vec; -use super::attribute::Attribute; +use super::attribute::{Attribute, Endianness}; use super::low_float::Format as LowFloatFormat; use super::nvfp4::ScaleApplication; use super::types::{Type, TypeKind, TypeTable}; @@ -44,22 +44,47 @@ impl Inst { } } +/// Binary operators. +/// +/// Integer types (`iN`) carry no signedness: a value is a bare bit pattern, and the +/// OPERATION decides how it is read. Operations whose result differs between a signed and +/// an unsigned reading therefore come in pairs (`UDiv`/`SDiv`, `Shr`/`Sar`, ...). +/// +/// Operand rules (enforced by `verify`; "int" and "float" include vectors of that lane type): +/// - `Add`, `Sub`, `Mul`: int or float (`Add`/`Sub` also pointer +/- int). +/// - `Div`, `Rem`: FLOAT ONLY (the IEEE division and remainder). +/// - `UDiv`, `SDiv`, `URem`, `SRem`: INT ONLY. Dividing by zero and `MIN / -1` behave exactly +/// as the backends and the emulator implement them. +/// - `BitAnd`, `BitOr`, `BitXor`: int or bool. +/// - `Shl`, `Shr` (logical right shift), `Sar` (arithmetic right shift): INT ONLY. +/// - `UMulh`, `SMulh`: INT ONLY; the high half of the double-width unsigned / signed product. +/// +/// Bitcode-relevant: `as u8` is the wire tag and the values are pinned. #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +#[repr(u8)] pub enum BinOp { - Add, - Sub, - Mul, - Div, - Rem, - BitAnd, - BitOr, - BitXor, - Shl, - Shr, - Mulh, + Add = 0, + Sub = 1, + Mul = 2, + Div = 3, + Rem = 4, + BitAnd = 5, + BitOr = 6, + BitXor = 7, + Shl = 8, + Shr = 9, + Sar = 10, + UDiv = 11, + SDiv = 12, + URem = 13, + SRem = 14, + UMulh = 15, + SMulh = 16, } impl BinOp { + /// The operator spelling of the text syntax (see the `parser` module comment). The + /// signedness suffix is `u` / `s`. pub fn symbol(self) -> &'static str { match self { BinOp::Add => "+", @@ -67,12 +92,18 @@ impl BinOp { BinOp::Mul => "*", BinOp::Div => "/", BinOp::Rem => "%", + BinOp::UDiv => "/u", + BinOp::SDiv => "/s", + BinOp::URem => "%u", + BinOp::SRem => "%s", BinOp::BitAnd => "&", BinOp::BitOr => "|", BinOp::BitXor => "^", BinOp::Shl => "<<", BinOp::Shr => ">>", - BinOp::Mulh => "*h", + BinOp::Sar => ">>s", + BinOp::UMulh => "*hu", + BinOp::SMulh => "*hs", } } } @@ -84,6 +115,9 @@ pub struct Arith { pub rhs: Value, } +/// `op lhs, imm` with the constant `imm` standing in for the right operand. The constant is +/// the one an `Iconst(imm)` of `lhs`'s type would be, so only its low `bits` matter; for a +/// shift it is the shift amount. `op` obeys the same rules as in [`Arith`]. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct ArithImm { pub op: BinOp, @@ -91,17 +125,37 @@ pub struct ArithImm { pub imm: i64, } +/// Comparison operators; the result is `bool` (for vector operands, a vector of `bool`). +/// +/// Operand rules (enforced by `verify`): both operands have the same type. +/// - `Eq`, `Ne`: any scalar (`bool`, int, float, ptr) or a vector of one. +/// - `Lt`, `Le`, `Gt`, `Ge`: FLOAT ONLY (ordered; false when either operand is NaN). +/// - `Slt`, `Sle`, `Sgt`, `Sge`: INT ONLY, the operands read as two's complement signed. +/// - `Ult`, `Ule`, `Ugt`, `Uge`: int or ptr, the operands read as unsigned. +/// +/// Bitcode-relevant: `as u8` is the wire tag and the values are pinned. #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +#[repr(u8)] pub enum CmpOp { - Eq, - Ne, - Lt, - Le, - Gt, - Ge, + Eq = 0, + Ne = 1, + Lt = 2, + Le = 3, + Gt = 4, + Ge = 5, + Slt = 6, + Sle = 7, + Sgt = 8, + Sge = 9, + Ult = 10, + Ule = 11, + Ugt = 12, + Uge = 13, } impl CmpOp { + /// The operator spelling of the text syntax (see the `parser` module comment). The + /// signedness suffix is `u` / `s`. pub fn symbol(self) -> &'static str { match self { CmpOp::Eq => "==", @@ -110,6 +164,14 @@ impl CmpOp { CmpOp::Le => "<=", CmpOp::Gt => ">", CmpOp::Ge => ">=", + CmpOp::Slt => " "<=s", + CmpOp::Sgt => ">s", + CmpOp::Sge => ">=s", + CmpOp::Ult => " "<=u", + CmpOp::Ugt => ">u", + CmpOp::Uge => ">=u", } } } @@ -139,9 +201,56 @@ pub struct Extract { pub index: u32, } +/// The conversion a [`Convert`] performs. `SiToFp` etc. name the source/destination class and +/// the SIGN of the integer side; widths come from the operand and result types. +/// +/// Type rules (enforced by `verify`; lane counts must match for vectors): +/// - `Trunc`: int -> strictly narrower int. Keeps the low bits. +/// - `Zext`: int or `bool` -> strictly wider int, filling with zeros. +/// - `Sext`: int -> strictly wider int, filling with the sign bit. +/// - `SiToFp` / `UiToFp`: int (read signed / unsigned) -> float. +/// - `FpToSi` / `FpToUi`: float -> int. In-range values truncate toward zero. The result of +/// an out-of-range or NaN input is UNSPECIFIED and differs between backends (aarch64 +/// saturates, x86-64 returns its "integer indefinite" value); frontends that need a +/// specific behavior must range-check or clamp the input first. +/// - `FpResize`: float -> float of a DIFFERENT width (extend or truncate). +/// +/// Same-width int <-> int casts have no kind: the value is simply reused. +/// +/// Bitcode-relevant: encoded as one byte, tag values pinned. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +#[repr(u8)] +pub enum ConvertKind { + Trunc = 0, + Zext = 1, + Sext = 2, + SiToFp = 3, + UiToFp = 4, + FpToSi = 5, + FpToUi = 6, + FpResize = 7, +} + +impl ConvertKind { + /// The mnemonic word of the text syntax: `convert , `. + pub fn name(self) -> &'static str { + match self { + ConvertKind::Trunc => "trunc", + ConvertKind::Zext => "zext", + ConvertKind::Sext => "sext", + ConvertKind::SiToFp => "sitofp", + ConvertKind::UiToFp => "uitofp", + ConvertKind::FpToSi => "fptosi", + ConvertKind::FpToUi => "fptoui", + ConvertKind::FpResize => "fpresize", + } + } +} + #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Convert { pub value: Value, + pub kind: ConvertKind, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -257,17 +366,69 @@ pub struct GlobalAddr { pub via_got: bool, } +/// Memory-access flags shared by [`Load`] and [`Store`]. +/// +/// Bitcode-relevant: see the `MemFlags` encoding in `bitcode.rs`. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub struct MemFlags { + /// The access must not be removed, merged or reordered by optimizers. + pub volatile: bool, + /// Guaranteed alignment of the address in bytes (power of two); 0 = no promise. + pub align: u32, + /// None = ordinary access. Some(o) = atomic load/store with ordering o. + /// Loads allow Relaxed|Acquire|SeqCst; stores allow Relaxed|Release|SeqCst. + pub ordering: Option, + /// Byte order of the memory image. `Native` and `Little` are identical on all current hosts + /// (both backends are little-endian); `Big` requires a byte swap. + pub endian: Endianness, +} + +impl Default for MemFlags { + fn default() -> Self { + Self::new() + } +} + +impl MemFlags { + pub const fn new() -> Self { + Self { + volatile: false, + align: 0, + ordering: None, + endian: Endianness::Native, + } + } + + /// True when the access needs nothing beyond a plain native load/store: + /// not volatile, not atomic, not big-endian. `align` is ignored. + pub fn is_plain(&self) -> bool { + !self.volatile && self.ordering.is_none() && self.endian != Endianness::Big + } +} + #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Load { pub ptr: Value, - pub volatile: bool, + pub mem: MemFlags, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Store { pub value: Value, pub ptr: Value, - pub volatile: bool, + pub mem: MemFlags, +} + +/// Open, target-defined operation identified by an interned name +/// (e.g. `aarch64.aese`). The result, if any, is typed by the instruction. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Intrinsic { + pub symbol: u32, + pub args: ValueList, + /// One immediate operand (lane index, rounding mode, ...). + pub imm: i64, + /// false = pure (may be removed if the result is unused, or duplicated). + pub side_effects: bool, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -291,11 +452,15 @@ pub struct VaEnd { pub list: Value, } +/// Dot-product accumulate: `acc + sum(a[i] * b[i])` over the lanes. For integer lanes +/// `signed` picks signed (`sdot`) or unsigned (`udot`) multiplication of the lanes; it has no +/// effect on float lanes (the text syntax always spells it, so a parser needs one form). #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Dot { pub acc: Value, pub a: Value, pub b: Value, + pub signed: bool, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -375,18 +540,21 @@ pub struct Barrier { pub scope: BarrierScope, } -/// Bitcode-relevant: names map 1:1 onto NVIDIA encode ops, order pinned. +/// Bitcode-relevant: names map 1:1 onto NVIDIA encode ops, tag values pinned. `SMin`/`SMax` +/// read the operands as signed integers, `UMin`/`UMax` as unsigned. #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] #[repr(u8)] pub enum AtomicOp { Add = 0, - Min = 1, - Max = 2, + SMin = 1, + SMax = 2, BitAnd = 3, BitOr = 4, BitXor = 5, Exchange = 6, CompareExchange = 7, + UMin = 8, + UMax = 9, } /// Bitcode-relevant: encoded as one byte, tag values pinned. @@ -542,6 +710,9 @@ impl<'a> AttrIterator<'a> { #[derive(Clone, PartialEq, Debug)] pub enum Opcode { + /// An integer constant. Integers are signless, so only the low `bits` of the payload + /// matter; `legalize` rewrites the payload of every sub-64-bit constant to the + /// zero-extended form the backends hold values in. Iconst(i64), Fconst(f64), Fconst128(u128), @@ -573,6 +744,7 @@ pub enum Opcode { Matmul(MatMul), Barrier(Barrier), AtomicRmw(AtomicRmw), + Intrinsic(Intrinsic), If(If), } @@ -708,24 +880,6 @@ impl Function { } } - pub fn is_byte_order_tagged(&self, inst: Inst) -> bool { - let mut on_inst = self.attributes_of(AttrTarget::Inst(inst)); - while let Some(attr) = on_inst.next_attr() { - if matches!(attr, Attribute::Endian(_)) { - return true; - } - } - if let Some(result) = self.inst_result(inst) { - let mut on_result = self.attributes_of(AttrTarget::Value(result)); - while let Some(attr) = on_result.next_attr() { - if matches!(attr, Attribute::Endian(_)) { - return true; - } - } - } - false - } - pub fn clone_attrs(&mut self, values: &[ValuePair], insts: &[InstPair]) { let end = self.attributes.len(); for i in 0..end { @@ -868,18 +1022,55 @@ impl Function { } pub fn append_store(&mut self, block: Block, value: Value, ptr: Value) { - self.append_store_vol(block, value, ptr, false); + self.append_store_mem(block, value, ptr, MemFlags::new()); } - pub fn append_store_vol(&mut self, block: Block, value: Value, ptr: Value, volatile: bool) { - self.append_stmt( - block, - Opcode::Store(Store { - value, - ptr, - volatile, - }), - ); + pub fn append_store_mem(&mut self, block: Block, value: Value, ptr: Value, mem: MemFlags) { + self.append_stmt(block, Opcode::Store(Store { value, ptr, mem })); + } + + pub fn append_load(&mut self, block: Block, ty: Type, ptr: Value, mem: MemFlags) -> Value { + self.append_inst(block, ty, Opcode::Load(Load { ptr, mem })) + } + + pub fn append_intrinsic( + &mut self, + block: Block, + ty: Type, + name: &str, + args: &[Value], + imm: i64, + side_effects: bool, + ) -> Value { + let intrinsic = self.make_intrinsic(name, args, imm, side_effects); + self.append_inst(block, ty, Opcode::Intrinsic(intrinsic)) + } + + pub fn append_intrinsic_void( + &mut self, + block: Block, + name: &str, + args: &[Value], + imm: i64, + side_effects: bool, + ) { + let intrinsic = self.make_intrinsic(name, args, imm, side_effects); + self.append_stmt(block, Opcode::Intrinsic(intrinsic)); + } + + fn make_intrinsic( + &mut self, + name: &str, + args: &[Value], + imm: i64, + side_effects: bool, + ) -> Intrinsic { + Intrinsic { + symbol: self.intern_symbol(name), + args: self.intern_values(args), + imm, + side_effects, + } } pub fn append_prefetch(&mut self, block: Block, ptr: Value) { @@ -911,9 +1102,26 @@ impl Function { self.append_stmt(block, Opcode::AtomicRmw(rmw)); } - pub fn append_dot(&mut self, block: Block, acc: Value, a: Value, b: Value) -> Value { + pub fn append_dot( + &mut self, + block: Block, + acc: Value, + a: Value, + b: Value, + signed: bool, + ) -> Value { let ty = self.value_type(acc); - self.append_inst(block, ty, Opcode::Dot(Dot { acc, a, b })) + self.append_inst(block, ty, Opcode::Dot(Dot { acc, a, b, signed })) + } + + pub fn append_convert( + &mut self, + block: Block, + ty: Type, + kind: ConvertKind, + value: Value, + ) -> Value { + self.append_inst(block, ty, Opcode::Convert(Convert { value, kind })) } pub fn append_reduce(&mut self, block: Block, op: BinOp, vector: Value) -> Value { @@ -1346,6 +1554,7 @@ impl Function { match &inst.op { Opcode::StructNew(sn) => lists.push((sn.fields.start, sn.fields.len)), Opcode::Call(c) => lists.push((c.args.start, c.args.len)), + Opcode::Intrinsic(i) => lists.push((i.args.start, i.args.len)), Opcode::CallIndirect(c) => lists.push((c.args.start, c.args.len)), Opcode::If(cf) => { lists.push((cf.then.args.start, cf.then.args.len)); @@ -1416,6 +1625,7 @@ impl Function { } } Opcode::StructNew(_) + | Opcode::Intrinsic(_) | Opcode::Call(_) | Opcode::CallIndirect(_) | Opcode::If(_) => {} @@ -1632,6 +1842,7 @@ impl Function { }), Opcode::Convert(cv) => Opcode::Convert(Convert { value: remap(map, cv.value), + kind: cv.kind, }), Opcode::DecodeLowFloat(cv) => Opcode::DecodeLowFloat(LowFloatConvert { value: remap(map, cv.value), @@ -1676,12 +1887,12 @@ impl Function { } Opcode::Load(ld) => Opcode::Load(Load { ptr: remap(map, ld.ptr), - volatile: ld.volatile, + mem: ld.mem, }), Opcode::Store(st) => Opcode::Store(Store { value: remap(map, st.value), ptr: remap(map, st.ptr), - volatile: st.volatile, + mem: st.mem, }), Opcode::Prefetch(pf) => Opcode::Prefetch(Prefetch { ptr: remap(map, pf.ptr), @@ -1700,6 +1911,7 @@ impl Function { acc: remap(map, d.acc), a: remap(map, d.a), b: remap(map, d.b), + signed: d.signed, }), Opcode::Reduce(red) => Opcode::Reduce(Reduce { vector: remap(map, red.vector), @@ -1722,6 +1934,10 @@ impl Function { } Opcode::AtomicRmw(a) } + Opcode::Intrinsic(mut i) => { + i.args = self.remap_value_list(i.args, map); + Opcode::Intrinsic(i) + } Opcode::If(cond) => Opcode::If(If { cond: remap(map, cond.cond), then: Jump { diff --git a/src/ir/src/legalize.rs b/src/ir/src/legalize.rs index 052bfef..b1cfcf5 100644 --- a/src/ir/src/legalize.rs +++ b/src/ir/src/legalize.rs @@ -2,19 +2,262 @@ extern crate alloc; use alloc::vec::Vec; -use super::function::{BinOp, Block, Function, Opcode, Value}; -use super::types::TypeKind; +use super::attribute::Endianness; +use super::function::{ + Arith, BinOp, Block, Function, Inst, Load, MemFlags, Opcode, Store, Unary, UnaryOp, Value, +}; +use super::types::{IntDesc, Type, TypeKind}; +use super::verify::mem_flags_valid; +/// Lowers the IR to what the native backends select instructions for. +/// +/// Besides struct-param splitting and folding, this removes every big-endian `Load`/`Store` +/// (afterwards `mem.endian` is in {Native, Little}) and every atomic-ordered float access. +/// Atomic-ordered int and pointer accesses stay as `Load`/`Store` with `mem.ordering` set: the +/// backends lower those natively. See [`legalize_memory`] for the rewrites and the +/// little-endian host assumption. +/// +/// Integer constants are signless, so `-1` and `0xffff_ffff` are the same `i32`; this rewrites +/// every sub-64-bit `Iconst` to the canonical zero-extended payload (the form the backends hold +/// values in) and folds constants and immediates width-aware, with the signedness taken from +/// the operation (`UDiv` vs `SDiv`, ...), never from a type. pub fn legalize(func: &mut Function) { split_struct_params(func); let mut subst: Vec<(Value, Value)> = Vec::new(); build_subst(func, &mut subst); apply_subst(func, &subst); + legalize_memory(func); + canonicalize_constants(func); fold_constants(func); fold_immediates(func); drop_dead(func); } +// --- Memory legalization ------------------------------------------------------- + +/// Rewrites big-endian loads/stores and atomic-ordered float accesses into forms the backends +/// select instructions for. +/// +/// ASSUMES A LITTLE-ENDIAN HOST: `Native` and `Little` are treated as identical, and +/// `Big` is realised as a byte swap around a native access. +/// +/// - Atomic int/pointer access -> unchanged: it stays a `Load`/`Store` carrying `mem.ordering` +/// (pointers are 8-byte scalars to the backends, so they pass through like a `u64`). +/// - Atomic float access -> the same-width int access (still atomic) plus +/// `Unary::Reinterpret`, since the backends only order integer-register accesses. +/// - Big endian -> byte swap (shifts, masks, ors in the same int type) after a load / +/// before a store; composed around the (possibly atomic) access when both are requested. +/// Float and pointer accesses go through the same-width int with +/// `Unary::Reinterpret`. +/// +/// `volatile`, `align` and `ordering` stay on the resulting access. Accesses the verifier +/// rejects are left untouched so backends report them. +fn legalize_memory(func: &mut Function) { + for bi in 0..func.block_count() { + let block = Block::from_u32(bi as u32); + let insts = func.block_insts(block).to_vec(); + let mut out: Vec = Vec::with_capacity(insts.len()); + let mut changed = false; + for inst in insts { + match func.opcode(inst) { + Opcode::Load(l) if needs_lowering(&l.mem) => { + changed |= lower_load(func, &mut out, inst, l); + } + Opcode::Store(st) if needs_lowering(&st.mem) => { + changed |= lower_store(func, &mut out, inst, st); + } + _ => {} + } + out.push(inst); + } + if changed { + func.set_block_insts(block, &out); + } + } +} + +fn needs_lowering(mem: &MemFlags) -> bool { + mem.ordering.is_some() || mem.endian == Endianness::Big +} + +/// The integer type an access of `ty` is performed in (`ty` itself when it is an int). +fn access_int_type(func: &mut Function, ty: Type, bytes: u32) -> Type { + match func.types.type_kind(ty) { + TypeKind::Int(_) => ty, + _ => func.types.intern(TypeKind::Int(IntDesc { + bits: (bytes * 8) as u16, + })), + } +} + +fn emit(func: &mut Function, out: &mut Vec, ty: Type, op: Opcode) -> Value { + let value = func.create_inst(ty, op); + out.push(func.defining_inst(value).unwrap()); + value +} + +/// An integer constant of `ty`, wrapped to the type's width and zero-extended (the canonical +/// payload, see [`canonicalize_constants`]). +fn int_const(func: &mut Function, out: &mut Vec, ty: Type, value: u64) -> Value { + let wrapped = match func.types.type_kind(ty) { + TypeKind::Int(d) => zero_extend(value as i64, u32::from(d.bits)), + _ => value as i64, + }; + emit(func, out, ty, Opcode::Iconst(wrapped)) +} + +fn arith( + func: &mut Function, + out: &mut Vec, + ty: Type, + op: BinOp, + lhs: Value, + rhs: Value, +) -> Value { + emit(func, out, ty, Opcode::Arith(Arith { op, lhs, rhs })) +} + +/// Byte `i` of `x` moved to byte position `bytes - 1 - i`. +fn swapped_byte( + func: &mut Function, + out: &mut Vec, + x: Value, + ty: Type, + bytes: u32, + i: u32, +) -> Value { + let j = bytes - 1 - i; + if i < j { + let mask = int_const(func, out, ty, 0xffu64 << (8 * i)); + let low = arith(func, out, ty, BinOp::BitAnd, x, mask); + let amount = int_const(func, out, ty, u64::from(8 * (j - i))); + arith(func, out, ty, BinOp::Shl, low, amount) + } else { + let amount = int_const(func, out, ty, u64::from(8 * (i - j))); + let shifted = arith(func, out, ty, BinOp::Shr, x, amount); + let mask = int_const(func, out, ty, 0xffu64 << (8 * j)); + arith(func, out, ty, BinOp::BitAnd, shifted, mask) + } +} + +/// Emits a byte swap of the `bytes`-wide (2, 4 or 8) int `x` of type `ty` and returns its +/// FINAL operation unemitted, so the caller can place it onto the instruction it replaces. +/// Every term is masked after a right shift, so the (logical) `Shr` is all that is needed. +fn byte_swap(func: &mut Function, out: &mut Vec, x: Value, ty: Type, bytes: u32) -> Opcode { + let mut acc = swapped_byte(func, out, x, ty, bytes, 0); + for i in 1..bytes - 1 { + let term = swapped_byte(func, out, x, ty, bytes, i); + acc = arith(func, out, ty, BinOp::BitOr, acc, term); + } + let term = swapped_byte(func, out, x, ty, bytes, bytes - 1); + Opcode::Arith(Arith { + op: BinOp::BitOr, + lhs: acc, + rhs: term, + }) +} + +fn reinterpret(value: Value) -> Opcode { + Opcode::Unary(Unary { + op: UnaryOp::Reinterpret, + value, + }) +} + +/// `mem` with the byte order lowered away (the byte swap is emitted separately). +fn native_mem(mem: MemFlags) -> MemFlags { + MemFlags { + endian: Endianness::Native, + ..mem + } +} + +fn is_float(func: &Function, ty: Type) -> bool { + matches!(func.types.type_kind(ty), TypeKind::Float(_)) +} + +fn lower_load(func: &mut Function, out: &mut Vec, inst: Inst, l: Load) -> bool { + let Some(result) = func.inst_result(inst) else { + return false; + }; + let ty = func.value_type(result); + if !mem_flags_valid(func, &l.mem, ty, false) { + return false; + } + let Some(bytes) = func.types.scalar_mem_bytes(ty) else { + return false; + }; + let big = l.mem.endian == Endianness::Big && bytes > 1; + let atomic_float = l.mem.ordering.is_some() && is_float(func, ty); + if !big && !atomic_float { + // Nothing to swap or reinterpret: an ordered int/ptr load is native, a one-byte + // big-endian one is just a native load. + if l.mem.endian == Endianness::Big { + *func.opcode_mut(inst) = Opcode::Load(Load { + mem: native_mem(l.mem), + ..l + }); + return true; + } + return false; + } + let int_ty = access_int_type(func, ty, bytes); + + // `pending` always yields the access in `int_ty`; the last step lands on `inst`. + let mut pending = Opcode::Load(Load { + ptr: l.ptr, + mem: native_mem(l.mem), + }); + if big { + let raw = emit(func, out, int_ty, pending); + pending = byte_swap(func, out, raw, int_ty, bytes); + } + if int_ty != ty { + let int_value = emit(func, out, int_ty, pending); + pending = reinterpret(int_value); + } + *func.opcode_mut(inst) = pending; + true +} + +fn lower_store(func: &mut Function, out: &mut Vec, inst: Inst, st: Store) -> bool { + let ty = func.value_type(st.value); + if !mem_flags_valid(func, &st.mem, ty, true) { + return false; + } + let Some(bytes) = func.types.scalar_mem_bytes(ty) else { + return false; + }; + let big = st.mem.endian == Endianness::Big && bytes > 1; + let atomic_float = st.mem.ordering.is_some() && is_float(func, ty); + if !big && !atomic_float { + if st.mem.endian == Endianness::Big { + *func.opcode_mut(inst) = Opcode::Store(Store { + mem: native_mem(st.mem), + ..st + }); + return true; + } + return false; + } + let int_ty = access_int_type(func, ty, bytes); + + let mut value = st.value; + if int_ty != ty { + value = emit(func, out, int_ty, reinterpret(value)); + } + if big { + let swap = byte_swap(func, out, value, int_ty, bytes); + value = emit(func, out, int_ty, swap); + } + *func.opcode_mut(inst) = Opcode::Store(Store { + value, + ptr: st.ptr, + mem: native_mem(st.mem), + }); + true +} + fn split_struct_params(func: &mut Function) { // Split struct-typed block params into scalar field params; rewrite // jump/if edges by extracting fields from the struct argument. @@ -337,6 +580,12 @@ fn apply_subst(func: &mut Function, subst: &[(Value, Value)]) { cf.else_.args = func.intern_values(&oe); Opcode::If(cf) } + Opcode::Intrinsic(mut i) => { + let vals = func.value_list(i.args).to_vec(); + let out: Vec = vals.iter().map(|v| sub(subst, *v)).collect(); + i.args = func.intern_values(&out); + Opcode::Intrinsic(i) + } Opcode::StructNew(sn) => { let vals = func.value_list(sn.fields).to_vec(); let out: Vec = vals.iter().map(|v| sub(subst, *v)).collect(); @@ -381,32 +630,91 @@ fn const_value(func: &Function, v: Value) -> Option { } } -fn fold_arith(op: BinOp, lhs: i64, rhs: i64) -> Option { - match op { - BinOp::Add => Some(lhs.wrapping_add(rhs)), - BinOp::Sub => Some(lhs.wrapping_sub(rhs)), - BinOp::Mul => Some(lhs.wrapping_mul(rhs)), - BinOp::Div => { - if rhs == 0 { - None - } else { - Some(lhs.wrapping_div(rhs)) - } - } - BinOp::Rem => { - if rhs == 0 { - None - } else { - Some(lhs.wrapping_rem(rhs)) +/// `value` wrapped to `bits` and zero-extended: the canonical payload of a `bits`-wide integer. +fn zero_extend(value: i64, bits: u32) -> i64 { + if bits == 0 || bits >= 64 { + value + } else { + (value as u64 & ((1u64 << bits) - 1)) as i64 + } +} + +/// The low `bits` of `value` read as a two's complement signed integer. +fn sign_extend(value: i64, bits: u32) -> i64 { + if bits == 0 || bits >= 64 { + value + } else { + (((value as u64) << (64 - bits)) as i64) >> (64 - bits) + } +} + +/// The width of a constant of `v`'s type: an int's own, a pointer's 64, a bool's 1. +fn value_bits(func: &Function, v: Value) -> Option { + match func.types.type_kind(func.value_type(v)) { + TypeKind::Int(d) => Some(u32::from(d.bits)), + TypeKind::Ptr(_) => Some(64), + TypeKind::Bool => Some(1), + _ => None, + } +} + +/// Integers are signless, so a constant is only its low `bits`: rewrite every sub-64-bit +/// integer `Iconst` to exactly that, zero-extended. The backends hold a value narrower than 64 +/// bits zero-extended and need no further case analysis for a constant. +fn canonicalize_constants(func: &mut Function) { + for i in 0..func.inst_count() { + let inst = super::function::Inst::from_u32(i as u32); + let Opcode::Iconst(c) = func.opcode(inst) else { + continue; + }; + let Some(result) = func.inst_result(inst) else { + continue; + }; + if let TypeKind::Int(d) = func.types.type_kind(func.value_type(result)) { + let canonical = zero_extend(c, u32::from(d.bits)); + if canonical != c { + *func.opcode_mut(inst) = Opcode::Iconst(canonical); } } - BinOp::BitAnd => Some(lhs & rhs), - BinOp::BitOr => Some(lhs | rhs), - BinOp::BitXor => Some(lhs ^ rhs), - BinOp::Shl | BinOp::Shr | BinOp::Mulh => None, } } +/// The value of `lhs op rhs` on `bits`-wide constants, in canonical (zero-extended) form, or +/// `None` when the operation is left to run time: shifts and high multiplies are not folded, +/// and neither is a division by zero or the overflowing `MIN / -1`, so that the backend's own +/// behavior for them is kept. +fn fold_arith(op: BinOp, lhs: i64, rhs: i64, bits: u32) -> Option { + let (ul, ur) = (zero_extend(lhs, bits) as u64, zero_extend(rhs, bits) as u64); + let (sl, sr) = (sign_extend(lhs, bits), sign_extend(rhs, bits)); + let min = if bits == 0 || bits >= 64 { + i64::MIN + } else { + -(1i64 << (bits - 1)) + }; + let folded = match op { + BinOp::Add => lhs.wrapping_add(rhs), + BinOp::Sub => lhs.wrapping_sub(rhs), + BinOp::Mul => lhs.wrapping_mul(rhs), + BinOp::BitAnd => lhs & rhs, + BinOp::BitOr => lhs | rhs, + BinOp::BitXor => lhs ^ rhs, + BinOp::UDiv => ul.checked_div(ur)? as i64, + BinOp::URem => ul.checked_rem(ur)? as i64, + BinOp::SDiv | BinOp::SRem if sl == min && sr == -1 => return None, + BinOp::SDiv => sl.checked_div(sr)?, + BinOp::SRem => sl.checked_rem(sr)?, + // `Div` and `Rem` are floating point; the rest is not folded. + BinOp::Div + | BinOp::Rem + | BinOp::Shl + | BinOp::Shr + | BinOp::Sar + | BinOp::UMulh + | BinOp::SMulh => return None, + }; + Some(zero_extend(folded, bits)) +} + fn fold_constants(func: &mut Function) { for i in 0..func.inst_count() { let inst = super::function::Inst::from_u32(i as u32); @@ -417,16 +725,15 @@ fn fold_constants(func: &mut Function) { let (Some(l), Some(r)) = (const_value(func, lhs), const_value(func, rhs)) else { continue; }; - if let Some(folded) = fold_arith(op, l, r) { + let Some(bits) = value_bits(func, lhs) else { + continue; + }; + if let Some(folded) = fold_arith(op, l, r, bits) { *func.opcode_mut(inst) = Opcode::Iconst(folded); } } } -fn is_unsigned(func: &Function, v: Value) -> bool { - matches!(func.types.type_kind(func.value_type(v)), TypeKind::Int(i) if !i.signed) -} - fn power_of_two_shift(c: i64) -> Option { if c < 2 { return None; @@ -448,6 +755,7 @@ fn imm_has_form(op: BinOp) -> bool { | BinOp::BitXor | BinOp::Shl | BinOp::Shr + | BinOp::Sar ) } @@ -460,12 +768,23 @@ fn imm_commutative(op: BinOp) -> bool { fn imm_fits(op: BinOp, c: i64) -> bool { match op { - BinOp::Shl | BinOp::Shr => (0..=63).contains(&c), + BinOp::Shl | BinOp::Shr | BinOp::Sar => (0..=63).contains(&c), BinOp::Sub => (-2047..=2048).contains(&c), _ => (-2048..=2047).contains(&c), } } +/// The immediate `op` takes for the `bits`-wide constant `c`, if it has an immediate form that +/// fits. Addition and subtraction wrap, so a constant is as well written sign-extended: +/// `x + 0xffff_ffff` on an `i32` is `x + -1`. +fn imm_for(op: BinOp, c: i64, bits: u32) -> Option { + if imm_fits(op, c) { + return Some(c); + } + let signed = sign_extend(c, bits); + (matches!(op, BinOp::Add | BinOp::Sub) && imm_fits(op, signed)).then_some(signed) +} + fn fold_immediates(func: &mut Function) { for i in 0..func.inst_count() { let inst = super::function::Inst::from_u32(i as u32); @@ -493,11 +812,11 @@ fn fold_immediates(func: &mut Function) { } continue; } - if (a.op == BinOp::Div || a.op == BinOp::Rem) && is_unsigned(func, a.lhs) { + if matches!(a.op, BinOp::UDiv | BinOp::URem) { if let Some(c) = const_value(func, a.rhs) && let Some(k) = power_of_two_shift(c) { - if a.op == BinOp::Div { + if a.op == BinOp::UDiv { *func.opcode_mut(inst) = Opcode::ArithImm(super::function::ArithImm { op: BinOp::Shr, lhs: a.lhs, @@ -516,22 +835,23 @@ fn fold_immediates(func: &mut Function) { if !imm_has_form(a.op) { continue; } + let bits = value_bits(func, a.lhs).unwrap_or(64); if let Some(c) = const_value(func, a.rhs) { - if imm_fits(a.op, c) { + if let Some(imm) = imm_for(a.op, c, bits) { *func.opcode_mut(inst) = Opcode::ArithImm(super::function::ArithImm { op: a.op, lhs: a.lhs, - imm: c, + imm, }); } } else if imm_commutative(a.op) && let Some(c) = const_value(func, a.lhs) - && imm_fits(a.op, c) + && let Some(imm) = imm_for(a.op, c, bits) { *func.opcode_mut(inst) = Opcode::ArithImm(super::function::ArithImm { op: a.op, lhs: a.rhs, - imm: c, + imm, }); } } @@ -560,7 +880,7 @@ fn is_pure(op: &Opcode) -> bool { | Opcode::Dot(_) | Opcode::Reduce(_) | Opcode::Splat(_) - ) + ) || matches!(op, Opcode::Intrinsic(i) if !i.side_effects) } fn count_uses(func: &Function, uses: &mut [u32]) { @@ -658,6 +978,11 @@ fn count_uses(func: &Function, uses: &mut [u32]) { use_one(v); } } + Opcode::Intrinsic(i) => { + for v in func.value_list(i.args).to_vec() { + use_one(v); + } + } Opcode::Iconst(_) | Opcode::Fconst(_) | Opcode::Fconst128(_) @@ -711,3 +1036,692 @@ fn drop_dead(func: &mut Function) { } } } + +#[cfg(test)] +mod tests { + use alloc::vec; + use alloc::vec::Vec; + use std::collections::HashMap; + + use super::*; + use crate::function::{AtomicOrdering, Ret, Terminator}; + use crate::types::{AddressSpace, FloatKind}; + use crate::verify::{Profile, verify}; + + /// Straight-line interpreter over one block, enough to execute legalized memory + /// accesses. Values are raw bits masked to their type's width; the address of the + /// pointer parameter is 0. Asserts it never sees an access `legalize` must remove. + struct Machine { + mem: Vec, + values: HashMap, + } + + fn bits_of(func: &Function, v: Value) -> u32 { + match func.types.type_kind(func.value_type(v)) { + TypeKind::Int(d) => u32::from(d.bits), + TypeKind::Float(FloatKind::F16) => 16, + TypeKind::Float(FloatKind::F32) => 32, + TypeKind::Float(FloatKind::F64) => 64, + TypeKind::Ptr(_) => 64, + other => panic!("unexpected type {other:?}"), + } + } + + fn mask(bits: u32) -> u64 { + if bits >= 64 { !0 } else { (1u64 << bits) - 1 } + } + + impl Machine { + fn new(mem: &[u8]) -> Self { + Self { + mem: mem.to_vec(), + values: HashMap::new(), + } + } + + fn read(&self, addr: u64, bytes: u32) -> u64 { + let a = addr as usize; + (0..bytes as usize).fold(0u64, |acc, i| acc | u64::from(self.mem[a + i]) << (8 * i)) + } + + fn write(&mut self, addr: u64, bytes: u32, value: u64) { + for i in 0..bytes as usize { + self.mem[addr as usize + i] = (value >> (8 * i)) as u8; + } + } + + fn binary(func: &Function, lhs: Value, op: BinOp, l: u64, r: u64) -> u64 { + let bits = bits_of(func, lhs); + let raw = match op { + BinOp::BitAnd => l & r, + BinOp::BitOr => l | r, + BinOp::Shl => l << r, + BinOp::Shr => l >> r, + other => panic!("unexpected op {other:?}"), + }; + raw & mask(bits) + } + + fn run(&mut self, func: &Function, block: Block, args: &[(Value, u64)]) { + self.values.extend(args.iter().copied()); + for &inst in func.block_insts(block) { + let result = func.inst_result(inst); + let out = match func.opcode(inst) { + Opcode::Iconst(c) => Some((c as u64) & mask(bits_of(func, result.unwrap()))), + Opcode::Arith(a) => { + let (l, r) = (self.values[&a.lhs], self.values[&a.rhs]); + Some(Self::binary(func, a.lhs, a.op, l, r)) + } + Opcode::ArithImm(a) => { + let l = self.values[&a.lhs]; + Some(Self::binary(func, a.lhs, a.op, l, a.imm as u64)) + } + Opcode::Unary(u) => { + assert_eq!(u.op, UnaryOp::Reinterpret); + let bits = bits_of(func, result.unwrap()); + assert_eq!(bits, bits_of(func, u.value)); + Some(self.values[&u.value] & mask(bits)) + } + // On a single thread an ordered access is an ordinary one. + Opcode::Load(l) => { + assert!(l.mem.endian != Endianness::Big); + let bytes = bits_of(func, result.unwrap()) / 8; + Some(self.read(self.values[&l.ptr], bytes)) + } + Opcode::Store(st) => { + assert!(st.mem.endian != Endianness::Big); + let bytes = bits_of(func, st.value) / 8; + self.write(self.values[&st.ptr], bytes, self.values[&st.value]); + None + } + other => panic!("unexpected opcode {other:?}"), + }; + if let (Some(r), Some(v)) = (result, out) { + self.values.insert(r, v); + } + } + } + } + + fn int_ty(func: &mut Function, bits: u16) -> Type { + func.types.intern(TypeKind::Int(IntDesc { bits })) + } + + fn mem(ordering: Option, endian: Endianness) -> MemFlags { + MemFlags { + ordering, + endian, + ..MemFlags::new() + } + } + + /// `fn(p: ptr) -> ty { ret load. p }`; returns the function, entry block and loaded value. + fn load_fn(ty: TypeKind, m: MemFlags) -> (Function, Block, Value, Value) { + let mut func = Function::new(); + let ty = func.types.intern(ty); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let v = func.append_load(entry, ty, p, m); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + (func, entry, p, v) + } + + /// `fn(p: ptr, x: ty) { store. x, p }`. + fn store_fn(ty: TypeKind, m: MemFlags) -> (Function, Block, Value, Value) { + let mut func = Function::new(); + let ty = func.types.intern(ty); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let x = func.append_block_param(entry, ty); + func.append_store_mem(entry, x, p, m); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + (func, entry, p, x) + } + + fn assert_legal(func: &Function) { + let diags = verify(func, Profile::High); + assert!(diags.ok(), "{:?}", diags.items()); + for bi in 0..func.block_count() { + for &inst in func.block_insts(Block::from_u32(bi as u32)) { + match func.opcode(inst) { + Opcode::Load(l) => { + assert!(l.mem.endian != Endianness::Big); + if l.mem.ordering.is_some() { + assert!(!is_float( + func, + func.value_type(func.inst_result(inst).unwrap()) + )); + } + } + Opcode::Store(st) => { + assert!(st.mem.endian != Endianness::Big); + if st.mem.ordering.is_some() { + assert!(!is_float(func, func.value_type(st.value))); + } + } + _ => {} + } + } + } + } + + fn ops(func: &Function) -> Vec { + func.block_insts(Block::from_u32(0)) + .iter() + .map(|&i| func.opcode(i)) + .collect() + } + + fn int(bits: u16) -> TypeKind { + TypeKind::Int(IntDesc { bits }) + } + + #[test] + fn big_endian_loads_become_a_native_load_and_a_byte_swap() { + let image: [u8; 8] = [0xf1, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + for kind in [int(16), int(32), int(64)] { + let bytes = match &kind { + TypeKind::Int(d) => usize::from(d.bits / 8), + _ => unreachable!(), + }; + let (mut func, entry, p, v) = load_fn(kind.clone(), mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + let loads = ops(&func) + .into_iter() + .filter(|o| matches!(o, Opcode::Load(_))) + .count(); + assert_eq!(loads, 1, "{kind:?}"); + let mut m = Machine::new(&image); + m.run(&func, entry, &[(p, 0)]); + let want = image[..bytes] + .iter() + .fold(0u64, |acc, &b| acc << 8 | u64::from(b)); + assert_eq!(m.values[&v], want, "{kind:?}"); + } + } + + #[test] + fn big_endian_float_loads_swap_in_an_int_and_reinterpret() { + let (mut func, entry, p, v) = + load_fn(TypeKind::Float(FloatKind::F32), mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + assert!(matches!( + ops(&func).last(), + Some(Opcode::Unary(u)) if u.op == UnaryOp::Reinterpret + )); + let mut m = Machine::new(&1.5f32.to_bits().to_be_bytes()); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], u64::from(1.5f32.to_bits())); + } + + #[test] + fn big_endian_stores_swap_before_a_native_store() { + for (kind, value, want) in [ + (int(16), 0x0102u64, vec![0x01u8, 0x02]), + (int(32), 0x0102_0304, vec![1, 2, 3, 4]), + (int(64), 0x0102_0304_0506_0708, vec![1, 2, 3, 4, 5, 6, 7, 8]), + ] { + let (mut func, entry, p, x) = store_fn(kind, mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + let mut m = Machine::new(&[0xee; 8]); + m.run(&func, entry, &[(p, 0), (x, value)]); + assert_eq!(&m.mem[..want.len()], &want[..]); + assert!(m.mem[want.len()..].iter().all(|&b| b == 0xee)); + } + let (mut func, entry, p, x) = + store_fn(TypeKind::Float(FloatKind::F32), mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + let mut m = Machine::new(&[0; 4]); + m.run(&func, entry, &[(p, 0), (x, u64::from(2.5f32.to_bits()))]); + assert_eq!(m.mem, 2.5f32.to_bits().to_be_bytes()); + } + + #[test] + fn volatile_and_align_stay_on_the_plain_access() { + let flags = MemFlags { + volatile: true, + align: 4, + ordering: None, + endian: Endianness::Big, + }; + let (mut func, ..) = load_fn(int(32), flags); + legalize(&mut func); + let plain: Vec = ops(&func) + .into_iter() + .filter_map(|o| match o { + Opcode::Load(l) => Some(l.mem), + _ => None, + }) + .collect(); + assert_eq!( + plain, + [MemFlags { + volatile: true, + align: 4, + ordering: None, + endian: Endianness::Native, + }] + ); + let (mut func, ..) = store_fn(int(32), flags); + legalize(&mut func); + assert!(ops(&func).iter().any(|o| matches!( + o, + Opcode::Store(st) if st.mem.volatile && st.mem.align == 4 + && st.mem.endian == Endianness::Native + ))); + } + + #[test] + fn one_byte_big_endian_needs_no_swap() { + let (mut func, entry, p, v) = load_fn(int(8), mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + assert_eq!(ops(&func).len(), 1); + let mut m = Machine::new(&[0x7f]); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], 0x7f); + } + + #[test] + fn native_and_little_accesses_are_left_alone() { + for endian in [Endianness::Native, Endianness::Little] { + let flags = MemFlags { + volatile: true, + align: 8, + endian, + ..MemFlags::new() + }; + let (mut func, ..) = load_fn(int(64), flags); + let before = ops(&func); + legalize(&mut func); + assert_eq!(ops(&func), before); + } + } + + #[test] + fn atomic_int_loads_stay_ordered_loads() { + for ordering in [ + AtomicOrdering::Relaxed, + AtomicOrdering::Acquire, + AtomicOrdering::SeqCst, + ] { + for kind in [int(8), int(16), int(32), int(64)] { + let flags = mem(Some(ordering), Endianness::Native); + let (mut func, entry, p, v) = load_fn(kind.clone(), flags); + let before = ops(&func); + legalize(&mut func); + assert_legal(&func); + assert_eq!(ops(&func), before, "{kind:?} {ordering:?}"); + assert!(matches!( + ops(&func)[..], + [Opcode::Load(l)] if l.mem.ordering == Some(ordering) + )); + let mut m = Machine::new(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]); + m.run(&func, entry, &[(p, 0)]); + let bytes = match &kind { + TypeKind::Int(d) => u32::from(d.bits / 8), + _ => unreachable!(), + }; + assert_eq!(m.values[&v], m.read(0, bytes), "{kind:?}"); + assert_eq!( + &m.mem[..], + &[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88] + ); + } + } + } + + #[test] + fn atomic_float_loads_go_through_an_atomic_int_load() { + let seq = Some(AtomicOrdering::SeqCst); + let (mut func, entry, p, v) = load_fn( + TypeKind::Float(FloatKind::F64), + mem(seq, Endianness::Native), + ); + legalize(&mut func); + assert_legal(&func); + let ops = ops(&func); + assert_eq!(ops.len(), 2); + assert!(matches!( + ops[0], + Opcode::Load(l) if l.mem.ordering == seq + )); + assert!(matches!(ops[1], Opcode::Unary(u) if u.op == UnaryOp::Reinterpret)); + let mut m = Machine::new(&(-4.25f64).to_bits().to_le_bytes()); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], (-4.25f64).to_bits()); + } + + #[test] + fn atomic_pointer_accesses_pass_through_unchanged() { + let seq = Some(AtomicOrdering::SeqCst); + let (mut func, entry, p, v) = load_fn( + TypeKind::Ptr(AddressSpace::Global), + mem(seq, Endianness::Native), + ); + let before = ops(&func); + legalize(&mut func); + assert_legal(&func); + assert_eq!(ops(&func), before); + let mut m = Machine::new(&0x0123_4567_89ab_cdefu64.to_le_bytes()); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], 0x0123_4567_89ab_cdef); + + let (mut func, ..) = store_fn( + TypeKind::Ptr(AddressSpace::Global), + mem(Some(AtomicOrdering::Release), Endianness::Native), + ); + let before = ops(&func); + legalize(&mut func); + assert_eq!(ops(&func), before); + } + + #[test] + fn atomic_int_stores_stay_ordered_stores() { + for ordering in [ + AtomicOrdering::Relaxed, + AtomicOrdering::Release, + AtomicOrdering::SeqCst, + ] { + let (mut func, entry, p, x) = + store_fn(int(32), mem(Some(ordering), Endianness::Native)); + let before = ops(&func); + legalize(&mut func); + assert_legal(&func); + assert_eq!(ops(&func), before); + assert!(matches!( + ops(&func)[..], + [Opcode::Store(s)] if s.mem.ordering == Some(ordering) && s.ptr == p && s.value == x + )); + let mut m = Machine::new(&[0; 4]); + m.run(&func, entry, &[(p, 0), (x, 0xdead_beef)]); + assert_eq!(m.mem, 0xdead_beefu32.to_le_bytes()); + } + } + + #[test] + fn atomic_float_stores_reinterpret_into_an_atomic_int_store() { + let release = Some(AtomicOrdering::Release); + let (mut func, entry, p, x) = store_fn( + TypeKind::Float(FloatKind::F32), + mem(release, Endianness::Native), + ); + legalize(&mut func); + assert_legal(&func); + let ops = ops(&func); + assert_eq!(ops.len(), 2); + assert!(matches!(ops[0], Opcode::Unary(u) if u.op == UnaryOp::Reinterpret)); + assert!(matches!( + ops[1], + Opcode::Store(s) if s.mem.ordering == release + )); + let mut m = Machine::new(&[0; 4]); + m.run(&func, entry, &[(p, 0), (x, u64::from(0.5f32.to_bits()))]); + assert_eq!(m.mem, 0.5f32.to_bits().to_le_bytes()); + } + + #[test] + fn atomic_and_big_endian_compose() { + let flags = mem(Some(AtomicOrdering::Acquire), Endianness::Big); + let (mut func, entry, p, v) = load_fn(int(32), flags); + legalize(&mut func); + assert_legal(&func); + let mut m = Machine::new(&[0xde, 0xad, 0xbe, 0xef]); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], 0xdead_beef); + + let flags = mem(Some(AtomicOrdering::Release), Endianness::Big); + let (mut func, entry, p, x) = store_fn(int(32), flags); + legalize(&mut func); + assert_legal(&func); + assert!(ops(&func).iter().any(|o| matches!( + o, + Opcode::Store(s) if s.mem.ordering == Some(AtomicOrdering::Release) + && s.mem.endian == Endianness::Native + ))); + let mut m = Machine::new(&[0; 4]); + m.run(&func, entry, &[(p, 0), (x, 0xdead_beef)]); + assert_eq!(m.mem, [0xde, 0xad, 0xbe, 0xef]); + + let flags = mem(Some(AtomicOrdering::SeqCst), Endianness::Big); + let (mut func, entry, p, v) = load_fn(TypeKind::Float(FloatKind::F32), flags); + legalize(&mut func); + assert_legal(&func); + let mut m = Machine::new(&1.25f32.to_bits().to_be_bytes()); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], u64::from(1.25f32.to_bits())); + } + + #[test] + fn every_valid_flag_combination_legalizes_to_verifiable_plain_ops() { + use AtomicOrdering::*; + let mut func = Function::new(); + let i32_t = int_ty(&mut func, 32); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let mut last = None; + for volatile in [false, true] { + for align in [0, 4] { + for endian in [Endianness::Native, Endianness::Little, Endianness::Big] { + for (load_o, store_o) in [ + (None, None), + (Some(Relaxed), Some(Relaxed)), + (Some(Acquire), Some(Release)), + (Some(SeqCst), Some(SeqCst)), + ] { + let l = MemFlags { + volatile, + align, + ordering: load_o, + endian, + }; + let s = MemFlags { + volatile, + align, + ordering: store_o, + endian, + }; + let v = func.append_load(entry, i32_t, p, l); + func.append_store_mem(entry, v, p, s); + last = Some(v); + } + } + } + } + func.set_terminator(entry, Terminator::Ret(Ret::one(last.unwrap()))); + assert!(verify(&func, Profile::High).ok()); + legalize(&mut func); + assert_legal(&func); + } + + #[test] + fn accesses_the_verifier_rejects_are_left_untouched() { + // A release load: there is no sound lowering, so a backend must see it and refuse. + let (mut func, ..) = load_fn( + int(32), + mem(Some(AtomicOrdering::Release), Endianness::Native), + ); + let before = ops(&func); + legalize(&mut func); + assert_eq!(ops(&func), before); + // A big-endian aggregate. + let (mut func, ..) = load_fn(TypeKind::Struct(vec![]), mem(None, Endianness::Big)); + let before = ops(&func); + legalize(&mut func); + assert_eq!(ops(&func), before); + } + + #[test] + fn pure_unused_intrinsics_are_dropped_and_effectful_ones_kept() { + let mut func = Function::new(); + let i32_t = int_ty(&mut func, 32); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let _dead = func.append_intrinsic(entry, i32_t, "t.pure", &[a], 0, false); + let _kept = func.append_intrinsic(entry, i32_t, "t.effect", &[a], 0, true); + func.set_terminator(entry, Terminator::Ret(Ret::one(a))); + legalize(&mut func); + let names: Vec<&str> = ops(&func) + .iter() + .map(|o| match o { + Opcode::Intrinsic(i) => func.symbol_name(i.symbol), + _ => panic!("unexpected op"), + }) + .collect(); + assert_eq!(names, ["t.effect"]); + } + + /// The defining op of the value `fn() { ret }` returns after `legalize`, over + /// constants of `bits` width. + fn folded(bits: u16, op: BinOp, lhs: i64, rhs: i64) -> Opcode { + let mut func = Function::new(); + let ty = int_ty(&mut func, bits); + let entry = func.append_block(); + let a = func.append_inst(entry, ty, Opcode::Iconst(lhs)); + let b = func.append_inst(entry, ty, Opcode::Iconst(rhs)); + let v = func.append_inst(entry, ty, Opcode::Arith(Arith { op, lhs: a, rhs: b })); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + legalize(&mut func); + func.opcode(func.defining_inst(v).unwrap()) + } + + /// The defining op of the value `fn(x: i32) { ret x c }` returns after `legalize`. + fn with_constant(op: BinOp, c: i64) -> Opcode { + let mut func = Function::new(); + let ty = int_ty(&mut func, 32); + let entry = func.append_block(); + let x = func.append_block_param(entry, ty); + let k = func.append_inst(entry, ty, Opcode::Iconst(c)); + let v = func.append_inst(entry, ty, Opcode::Arith(Arith { op, lhs: x, rhs: k })); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + legalize(&mut func); + func.opcode(func.defining_inst(v).unwrap()) + } + + #[test] + fn integer_constants_become_their_zero_extended_bits() { + let mut func = Function::new(); + let (i8_t, i32_t, i64_t) = ( + int_ty(&mut func, 8), + int_ty(&mut func, 32), + int_ty(&mut func, 64), + ); + let entry = func.append_block(); + let minus_one = func.append_inst(entry, i32_t, Opcode::Iconst(-1)); + let wide = func.append_inst(entry, i8_t, Opcode::Iconst(0x1_0000_00ff)); + let all_ones = func.append_inst(entry, i64_t, Opcode::Iconst(-1)); + let min = func.append_inst(entry, i8_t, Opcode::Iconst(-128)); + func.set_terminator( + entry, + Terminator::Ret(Ret::many(&[minus_one, wide, all_ones, min])), + ); + legalize(&mut func); + let payload = |v: Value| match func.opcode(func.defining_inst(v).unwrap()) { + Opcode::Iconst(c) => c, + other => panic!("expected a constant, got {other:?}"), + }; + assert_eq!(payload(minus_one), 0xffff_ffff); + assert_eq!(payload(wide), 0xff); + assert_eq!(payload(all_ones), -1); + assert_eq!(payload(min), 0x80); + } + + #[test] + fn folding_reads_the_operands_as_the_operation_says() { + use BinOp::*; + // (bits, operator, lhs, rhs, the canonical zero-extended result) + let cases: &[(u16, BinOp, i64, i64, i64)] = &[ + (32, UDiv, -1, 2, 0x7fff_ffff), + (32, SDiv, -1, 2, 0), + (32, SDiv, -6, 2, 0xffff_fffd), + (32, URem, -1, 10, 5), + (32, SRem, -7, 3, 0xffff_ffff), + (8, UDiv, 200, 3, 66), + (8, SDiv, 200, 3, 0xee), + (8, SRem, 200, 3, 0xfe), + (8, Add, 200, 100, 44), + (8, Sub, 1, 2, 0xff), + (8, Mul, 16, 16, 0), + (16, BitXor, 0xffff, 0x00ff, 0xff00), + (64, UDiv, -1, 2, i64::MAX), + (64, SDiv, -1, 2, 0), + (64, URem, -1, 10, 5), + ]; + for &(bits, op, lhs, rhs, want) in cases { + match folded(bits, op, lhs, rhs) { + Opcode::Iconst(got) => { + assert_eq!(got, want, "i{bits} {op:?} {lhs} {rhs}"); + } + other => panic!("i{bits} {op:?} {lhs} {rhs} did not fold: {other:?}"), + } + } + } + + #[test] + fn folding_leaves_what_the_backend_defines_to_run_time() { + use BinOp::*; + let cases: &[(u16, BinOp, i64, i64)] = &[ + (32, UDiv, 5, 0), + (32, URem, 5, 0), + (32, SDiv, 5, 0), + (32, SRem, 5, 0), + (32, SDiv, i64::from(i32::MIN), -1), + (32, SRem, i64::from(i32::MIN), -1), + (8, SDiv, 0x80, 0xff), + (64, SDiv, i64::MIN, -1), + (64, SRem, i64::MIN, -1), + (32, UMulh, 3, 5), + (32, SMulh, 3, 5), + ]; + for &(bits, op, lhs, rhs) in cases { + assert!( + matches!(folded(bits, op, lhs, rhs), Opcode::Arith(a) if a.op == op), + "i{bits} {op:?} {lhs} {rhs}" + ); + } + } + + #[test] + fn immediates_follow_the_operation() { + use BinOp::*; + let imm = |op, c| match with_constant(op, c) { + Opcode::ArithImm(a) => { + assert_eq!(a.lhs, Value::from_u32(0), "{op:?} {c}"); + Some((a.op, a.imm)) + } + Opcode::Arith(a) => { + assert_eq!(a.op, op, "{op:?} {c}"); + None + } + other => panic!("unexpected {other:?}"), + }; + // Unsigned division and remainder by a power of two are a shift and a mask ... + assert_eq!(imm(UDiv, 8), Some((Shr, 3))); + assert_eq!(imm(URem, 8), Some((BitAnd, 7))); + assert_eq!(imm(URem, 4096), None); + // ... the signed forms and the high multiplies are not. + assert_eq!(imm(SDiv, 8), None); + assert_eq!(imm(SRem, 8), None); + assert_eq!(imm(UMulh, 2), None); + assert_eq!(imm(SMulh, 2), None); + // Both right shifts and the left shift have an immediate form. + assert_eq!(imm(Shr, 3), Some((Shr, 3))); + assert_eq!(imm(Sar, 3), Some((Sar, 3))); + assert_eq!(imm(Shl, 3), Some((Shl, 3))); + assert_eq!(imm(Sar, 64), None); + assert_eq!(imm(Mul, 16), Some((Shl, 4))); + // A constant that is -1 read signed is still `-1` for add / sub, which wrap. + assert_eq!(imm(Add, -1), Some((Add, -1))); + assert_eq!(imm(Sub, -1), Some((Sub, -1))); + assert_eq!(imm(Add, 0x7ff), Some((Add, 0x7ff))); + assert_eq!(imm(Add, 0x800), None); + assert_eq!(imm(BitAnd, 0xff), Some((BitAnd, 0xff))); + } +} diff --git a/src/ir/src/lib.rs b/src/ir/src/lib.rs index 9175cfe..c31f48b 100755 --- a/src/ir/src/lib.rs +++ b/src/ir/src/lib.rs @@ -1,17 +1,19 @@ -// NOTE: volt-ir requires std: pliron (and its dependencies) need std to -// build. no_std support is deferred until pliron can build without std. +#![no_std] extern crate alloc; +#[cfg(test)] +extern crate std; + pub mod attribute; pub mod bitcode; pub mod builder; pub mod critical_edge; -pub mod dialect; pub mod entity; pub mod expand; pub mod function; pub mod legalize; pub mod low_float; +pub mod module; pub mod nvfp4; pub mod parser; pub mod reachable; diff --git a/src/ir/src/module.rs b/src/ir/src/module.rs new file mode 100644 index 0000000..dca8063 --- /dev/null +++ b/src/ir/src/module.rs @@ -0,0 +1,505 @@ +//! A translation unit: named functions plus named data globals. +//! +//! [`Module`] is the single description of a unit of code and data shared by +//! every consumer (object writers, in-process linkers, JITs). Symbols that are +//! referenced but not defined here (external functions or data) are not an +//! error at this level; the linker or host resolves them. + +use alloc::string::{String, ToString as _}; +use alloc::vec::Vec; +use core::fmt; + +use crate::function::Function; +use crate::verify::{self, Diagnostic, Profile}; + +/// Size in bytes of a data relocation slot: a pointer-sized absolute address. +pub const RELOC_SLOT_SIZE: u64 = 8; + +/// The section a data global lands in: read-only, writable, or zero-initialized. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DataKind { + Rodata, + Data, + Bss, +} + +/// An internal relocation within a global. At byte `offset` within the global, +/// the linker writes the pointer-sized absolute runtime address of `symbol`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DataReloc { + pub offset: usize, + pub symbol: String, +} + +/// A named data global. For [`DataKind::Bss`], `bytes` is empty and `size` +/// gives the zero-initialized length; otherwise `size == bytes.len()`. +/// +/// `align` is the required alignment in bytes: `0` selects the natural +/// alignment ([`natural_align`] of `size`), otherwise it must be a power of two. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Global { + pub name: String, + pub kind: DataKind, + pub bytes: Vec, + pub size: u64, + pub align: u32, + pub relocs: Vec, +} + +impl Global { + /// The alignment this global is laid out with: `align`, or the natural + /// alignment of `size` when `align` is `0`. + pub fn effective_align(&self) -> usize { + match self.align { + 0 => natural_align(self.size), + a => a as usize, + } + } +} + +/// A data object's natural alignment: the next power of two up to `size`, +/// capped at 8. Covers scalars and pointers up to a 64-bit word. +pub fn natural_align(size: u64) -> usize { + size.max(1).checked_next_power_of_two().unwrap_or(8).min(8) as usize +} + +/// A named function definition. +#[derive(Clone, Debug)] +pub struct FunctionDef { + pub name: String, + pub function: Function, +} + +/// Why [`Module::verify`] rejected a module. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ModuleError { + /// Two definitions (function or global) share a name. + DuplicateName(String), + /// A non-bss global's `size` differs from its `bytes.len()`. + SizeMismatch { name: String, size: u64, len: usize }, + /// A bss global carries bytes or relocations. + BssInitialized(String), + /// A relocation slot does not fit within its global. + RelocOutOfRange { name: String, offset: usize }, + /// A global's `align` is neither `0` nor a power of two. + InvalidAlign { name: String, align: u32 }, + /// A function failed IR verification. + InvalidFunction { + name: String, + diagnostics: Vec, + }, +} + +impl fmt::Display for ModuleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DuplicateName(name) => write!(f, "duplicate module symbol {name}"), + Self::SizeMismatch { name, size, len } => { + write!( + f, + "global {name} has size {size} but {len} initializer bytes" + ) + } + Self::BssInitialized(name) => { + write!(f, "bss global {name} has initializer bytes or relocations") + } + Self::RelocOutOfRange { name, offset } => { + write!( + f, + "relocation at offset {offset} does not fit in global {name}" + ) + } + Self::InvalidAlign { name, align } => { + write!(f, "global {name} has invalid alignment {align}") + } + Self::InvalidFunction { name, diagnostics } => { + write!(f, "function {name} failed verification")?; + if let Some(first) = diagnostics.first() { + write!(f, ": {first:?}")?; + if diagnostics.len() > 1 { + write!(f, " (and {} more)", diagnostics.len() - 1)?; + } + } + Ok(()) + } + } + } +} + +impl core::error::Error for ModuleError {} + +/// A set of named functions and data globals. Function order is preserved; +/// linkers place the first function at the start of the code image. +#[derive(Clone, Debug, Default)] +pub struct Module { + pub functions: Vec, + pub globals: Vec, +} + +impl Module { + pub fn new() -> Module { + Module::default() + } + + pub fn add_function(&mut self, name: &str, function: Function) { + self.functions.push(FunctionDef { + name: name.to_string(), + function, + }); + } + + /// Add a named read-only data blob (a global constant). + pub fn add_data(&mut self, name: &str, bytes: Vec) { + self.add_data_relocs(name, bytes, Vec::new()); + } + + /// Add a named read-only data blob carrying internal relocations. + pub fn add_data_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { + self.add_initialized(name, DataKind::Rodata, bytes, relocs); + } + + /// Add a named writable data global. + pub fn add_writable(&mut self, name: &str, bytes: Vec) { + self.add_writable_relocs(name, bytes, Vec::new()); + } + + /// Add a named writable data global carrying internal relocations. + pub fn add_writable_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { + self.add_initialized(name, DataKind::Data, bytes, relocs); + } + + /// Add a named zero-initialized data global of `size` bytes. + pub fn add_bss(&mut self, name: &str, size: u64) { + self.globals.push(Global { + name: name.to_string(), + kind: DataKind::Bss, + bytes: Vec::new(), + size, + align: 0, + relocs: Vec::new(), + }); + } + + fn add_initialized( + &mut self, + name: &str, + kind: DataKind, + bytes: Vec, + relocs: Vec, + ) { + self.globals.push(Global { + name: name.to_string(), + kind, + size: bytes.len() as u64, + bytes, + align: 0, + relocs, + }); + } + + /// The function defined under `name`. + pub fn function(&self, name: &str) -> Option<&Function> { + self.functions + .iter() + .find(|f| f.name == name) + .map(|f| &f.function) + } + + /// The global defined under `name`. + pub fn global(&self, name: &str) -> Option<&Global> { + self.globals.iter().find(|g| g.name == name) + } + + /// Check module-level well-formedness and verify every function. + /// + /// Undefined symbols (calls, `global_addr`, data relocations naming + /// something not defined here) are allowed: they are external. + pub fn verify(&self) -> Result<(), ModuleError> { + let mut names: Vec<&str> = self + .functions + .iter() + .map(|f| f.name.as_str()) + .chain(self.globals.iter().map(|g| g.name.as_str())) + .collect(); + names.sort_unstable(); + if let Some(pair) = names.windows(2).find(|pair| pair[0] == pair[1]) { + return Err(ModuleError::DuplicateName(pair[0].to_string())); + } + for g in &self.globals { + self.verify_global(g)?; + } + for f in &self.functions { + let diagnostics = verify::verify(&f.function, Profile::High); + if !diagnostics.ok() { + return Err(ModuleError::InvalidFunction { + name: f.name.clone(), + diagnostics: diagnostics.items().to_vec(), + }); + } + } + Ok(()) + } + + fn verify_global(&self, g: &Global) -> Result<(), ModuleError> { + if g.align != 0 && !g.align.is_power_of_two() { + return Err(ModuleError::InvalidAlign { + name: g.name.clone(), + align: g.align, + }); + } + if g.kind == DataKind::Bss { + if !g.bytes.is_empty() || !g.relocs.is_empty() { + return Err(ModuleError::BssInitialized(g.name.clone())); + } + return Ok(()); + } + if g.size != g.bytes.len() as u64 { + return Err(ModuleError::SizeMismatch { + name: g.name.clone(), + size: g.size, + len: g.bytes.len(), + }); + } + for r in &g.relocs { + let fits = (r.offset as u64) + .checked_add(RELOC_SLOT_SIZE) + .is_some_and(|end| end <= g.size); + if !fits { + return Err(ModuleError::RelocOutOfRange { + name: g.name.clone(), + offset: r.offset, + }); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + use crate::function::{MemFlags, Ret, Terminator}; + + fn constant(value: i64) -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let v = f.append_inst(block, ty, crate::function::Opcode::Iconst(value)); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + f + } + + /// A function the verifier rejects: a load with a non-power-of-two `align`. + fn malformed() -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i32").unwrap(); + let ptr = f.types.ptr_global(); + let entry = f.append_block(); + let p = f.append_block_param(entry, ptr); + let mem = MemFlags { + align: 3, + ..MemFlags::new() + }; + let v = f.append_load(entry, ty, p, mem); + f.set_terminator(entry, Terminator::Ret(Ret::one(v))); + f + } + + #[test] + fn natural_alignment_covers_scalars() { + assert_eq!(natural_align(0), 1); + assert_eq!(natural_align(1), 1); + assert_eq!(natural_align(3), 4); + assert_eq!(natural_align(8), 8); + assert_eq!(natural_align(100), 8); + assert_eq!(natural_align(u64::MAX), 8); + } + + #[test] + fn builders_fill_kind_size_and_lookup() { + let mut m = Module::new(); + m.add_function("f", constant(1)); + m.add_data("ro", vec![1, 2, 3]); + m.add_writable_relocs( + "rw", + vec![0; 8], + vec![DataReloc { + offset: 0, + symbol: "f".to_string(), + }], + ); + m.add_bss("zero", 64); + assert!(m.function("f").is_some()); + assert!(m.function("ro").is_none()); + let ro = m.global("ro").unwrap(); + assert_eq!((ro.kind, ro.size, ro.align), (DataKind::Rodata, 3, 0)); + assert_eq!(ro.effective_align(), 4); + let rw = m.global("rw").unwrap(); + assert_eq!((rw.kind, rw.size, rw.relocs.len()), (DataKind::Data, 8, 1)); + let zero = m.global("zero").unwrap(); + assert_eq!( + (zero.kind, zero.size, zero.bytes.len()), + (DataKind::Bss, 64, 0) + ); + assert_eq!(zero.effective_align(), 8); + assert!(m.global("f").is_none()); + assert_eq!(m.verify(), Ok(())); + } + + #[test] + fn explicit_alignment_overrides_natural() { + let mut m = Module::new(); + m.add_bss("big", 64); + m.globals[0].align = 64; + assert_eq!(m.globals[0].effective_align(), 64); + assert_eq!(m.verify(), Ok(())); + } + + #[test] + fn undefined_symbols_are_external_not_errors() { + let mut m = Module::new(); + m.add_data_relocs( + "table", + vec![0; 16], + vec![DataReloc { + offset: 8, + symbol: "elsewhere".to_string(), + }], + ); + assert_eq!(m.verify(), Ok(())); + } + + #[test] + fn rejects_duplicate_names_across_functions_and_globals() { + let mut m = Module::new(); + m.add_function("x", constant(1)); + m.add_data("x", vec![0]); + assert_eq!(m.verify(), Err(ModuleError::DuplicateName("x".to_string()))); + + let mut m = Module::new(); + m.add_function("f", constant(1)); + m.add_function("f", constant(2)); + assert_eq!(m.verify(), Err(ModuleError::DuplicateName("f".to_string()))); + + let mut m = Module::new(); + m.add_bss("b", 8); + m.add_writable("b", vec![0]); + assert_eq!(m.verify(), Err(ModuleError::DuplicateName("b".to_string()))); + } + + #[test] + fn rejects_size_mismatch_and_initialized_bss() { + let mut m = Module::new(); + m.add_data("d", vec![0; 4]); + m.globals[0].size = 8; + assert_eq!( + m.verify(), + Err(ModuleError::SizeMismatch { + name: "d".to_string(), + size: 8, + len: 4 + }) + ); + + let mut m = Module::new(); + m.add_bss("z", 8); + m.globals[0].bytes = vec![0; 8]; + assert_eq!( + m.verify(), + Err(ModuleError::BssInitialized("z".to_string())) + ); + + let mut m = Module::new(); + m.add_bss("z", 16); + m.globals[0].relocs.push(DataReloc { + offset: 0, + symbol: "f".to_string(), + }); + assert_eq!( + m.verify(), + Err(ModuleError::BssInitialized("z".to_string())) + ); + } + + #[test] + fn relocation_slot_must_fit_in_the_global() { + let reloc = |offset| DataReloc { + offset, + symbol: "s".to_string(), + }; + let mut m = Module::new(); + m.add_data_relocs("fits", vec![0; 16], vec![reloc(0), reloc(8)]); + assert_eq!(m.verify(), Ok(())); + + for offset in [9, 16, usize::MAX] { + let mut m = Module::new(); + m.add_data_relocs("d", vec![0; 16], vec![reloc(offset)]); + assert_eq!( + m.verify(), + Err(ModuleError::RelocOutOfRange { + name: "d".to_string(), + offset + }), + "offset {offset}" + ); + } + let mut m = Module::new(); + m.add_data_relocs("short", vec![0; 4], vec![reloc(0)]); + assert!(matches!( + m.verify(), + Err(ModuleError::RelocOutOfRange { .. }) + )); + } + + #[test] + fn alignment_must_be_zero_or_a_power_of_two() { + for align in [0, 1, 2, 16, 4096] { + let mut m = Module::new(); + m.add_data("d", vec![0; 4]); + m.globals[0].align = align; + assert_eq!(m.verify(), Ok(()), "align {align}"); + } + for align in [3, 6, 12] { + let mut m = Module::new(); + m.add_data("d", vec![0; 4]); + m.globals[0].align = align; + assert_eq!( + m.verify(), + Err(ModuleError::InvalidAlign { + name: "d".to_string(), + align + }), + "align {align}" + ); + } + } + + #[test] + fn function_diagnostics_surface_in_the_error() { + let mut m = Module::new(); + m.add_function("ok", constant(1)); + m.add_function("bad", malformed()); + match m.verify() { + Err(ModuleError::InvalidFunction { name, diagnostics }) => { + assert_eq!(name, "bad"); + assert_eq!(diagnostics.len(), 1); + assert!(matches!(diagnostics[0], Diagnostic::InvalidMemFlags(_))); + } + other => panic!("expected InvalidFunction, got {other:?}"), + } + } + + #[test] + fn errors_display_and_implement_error() { + fn is_error(_: &E) {} + let e = ModuleError::DuplicateName("x".to_string()); + is_error(&e); + assert_eq!(e.to_string(), "duplicate module symbol x"); + let mut m = Module::new(); + m.add_function("bad", malformed()); + let text = m.verify().unwrap_err().to_string(); + assert!(text.starts_with("function bad failed verification: InvalidMemFlags")); + } +} diff --git a/src/ir/src/parser.rs b/src/ir/src/parser.rs index 90dc628..6d5cf81 100644 --- a/src/ir/src/parser.rs +++ b/src/ir/src/parser.rs @@ -2,6 +2,75 @@ //! of the function printer. Values and blocks are named positionally (`v{n}`, //! `block{n}`); the parser creates them in textual order and resolves //! references by number. Ported from `vulcan/libs/vulcan-ir/parser.zig`. +//! +//! # Memory accesses +//! +//! `load` and `store` take optional modifier words (the `MemFlags`) between the mnemonic +//! and the operands. Each may appear at most once, in any order; the canonical order +//! (the one a printer writes) is: +//! +//! ```text +//! let v1 = load [volatile] [atomic ] [align ] [be|le] , +//! store [volatile] [atomic ] [align ] [be|le] , +//! ``` +//! +//! - `volatile`: the access may not be removed, merged or reordered. +//! - `atomic `: `relaxed`, `acquire`, `release`, `seq_cst` (loads take +//! relaxed/acquire/seq_cst, stores relaxed/release/seq_cst; `acq_rel` parses but does not +//! verify on a load or store). +//! - `align `: the address is aligned to `n` bytes (a power of two). +//! - `be` / `le`: big / little endian memory image; omitted means native. +//! +//! # Signedness lives on the operation +//! +//! Integer types are plain `iN` (there is no `uN`); an operation decides whether it reads its +//! operands as signed or unsigned. Binary operators are infix (`let v2 = v0 v1`, or +//! `v0 ` for the immediate form); the signed / unsigned variant of an +//! operator is the same spelling with an `s` / `u` suffix: +//! +//! ```text +//! + - * & | ^ << >> add sub mul and or xor shl, logical shift right (integer) +//! / % FLOAT division / remainder only +//! /u /s %u %s unsigned / signed integer division / remainder +//! >>s arithmetic shift right (integer) +//! *hu *hs high half of the unsigned / signed double-width product +//! == != equality of any scalar +//! < <= > >= FLOAT ordered comparison only +//! s >=s signed integer comparison +//! u >=u unsigned integer / pointer comparison +//! ``` +//! +//! Conversions name the kind first, then the result type: +//! +//! ```text +//! let v1 = convert , v0 +//! ``` +//! +//! with `kind` one of `trunc zext sext sitofp uitofp fptosi fptoui fpresize` (the legal type +//! pairs of each are listed on `ConvertKind`). A same-width integer cast does not exist. +//! +//! `dot` and the atomic min/max spell the signedness as a word: +//! +//! ```text +//! let v3 = dot signed|unsigned v0, v1, v2 +//! let v2 = atomic_rmw add|smin|smax|umin|umax|bit_and|bit_or|bit_xor|exchange v0, v1 +//! ``` +//! +//! Words naming a `BinOp` (`reduce add, v0`) are `add sub mul div rem udiv sdiv urem srem +//! bit_and bit_or bit_xor shl shr sar umulh smulh`. +//! +//! # Intrinsics +//! +//! An intrinsic is a target-defined operation named by a dotted identifier: +//! +//! ```text +//! let v2 = intrinsic @aarch64.aese(v0, v1) [imm ] [pure] +//! intrinsic @aarch64.dmb() [imm ] [pure] +//! ``` +//! +//! `imm` is the one signed immediate operand (omitted means 0, and the printer omits it +//! then). `pure` marks the intrinsic free of side effects (`Intrinsic::side_effects == +//! false`); without it the intrinsic is assumed to have side effects. use alloc::string::{String, ToString}; use alloc::vec::Vec; @@ -9,10 +78,10 @@ use alloc::vec::Vec; use super::attribute::{AttrValue, Attribute, Custom, Endianness}; use super::function::{ Alloca, Arith, AtomicOp, AtomicOrdering, AtomicRmw, AtomicScope, AttrTarget, BarrierScope, - BinOp, Block, Call, CallIndirect, CmpOp, Compare, Convert, EdgeDesc, Extract, Function, - GlobalAddr, InputSigns, Load, LowFloatConvert, MatMul, MatMulQuant, MatMulQuantOut, - MatMulScale, MatMulType, NvFp4Convert, Opcode, Reduce, Ret, RetPiece, Select, Terminator, - Unary, UnaryOp, Value, + BinOp, Block, Call, CallIndirect, CmpOp, Compare, ConvertKind, EdgeDesc, Extract, Function, + GlobalAddr, InputSigns, LowFloatConvert, MatMul, MatMulQuant, MatMulQuantOut, MatMulScale, + MatMulType, MemFlags, NvFp4Convert, Opcode, Reduce, Ret, RetPiece, Select, Terminator, Unary, + UnaryOp, Value, }; use super::low_float::Format as LowFloatFormat; use super::nvfp4::ScaleApplication; @@ -42,20 +111,45 @@ impl From for ParseError { } } -/// Map a leading character to a binary arithmetic operator, if it is one. -fn arith_op_of(c: u8) -> Option { - match c { - b'+' => Some(BinOp::Add), - b'-' => Some(BinOp::Sub), - b'*' => Some(BinOp::Mul), - b'/' => Some(BinOp::Div), - b'%' => Some(BinOp::Rem), - b'&' => Some(BinOp::BitAnd), - b'|' => Some(BinOp::BitOr), - b'^' => Some(BinOp::BitXor), - _ => None, - } -} +/// The infix spellings of the binary operators. A spelling that is a prefix of another +/// (`>>` of `>>s`, `*` of `*hu`) comes after it, so the first match wins. +const ARITH_OPS: [(&str, BinOp); 17] = [ + ("<<", BinOp::Shl), + (">>s", BinOp::Sar), + (">>", BinOp::Shr), + ("*hu", BinOp::UMulh), + ("*hs", BinOp::SMulh), + ("/u", BinOp::UDiv), + ("/s", BinOp::SDiv), + ("%u", BinOp::URem), + ("%s", BinOp::SRem), + ("+", BinOp::Add), + ("-", BinOp::Sub), + ("*", BinOp::Mul), + ("/", BinOp::Div), + ("%", BinOp::Rem), + ("&", BinOp::BitAnd), + ("|", BinOp::BitOr), + ("^", BinOp::BitXor), +]; + +/// The infix spellings of the comparisons, ordered like [`ARITH_OPS`]. +const CMP_OPS: [(&str, CmpOp); 14] = [ + ("==", CmpOp::Eq), + ("!=", CmpOp::Ne), + ("<=s", CmpOp::Sle), + ("<=u", CmpOp::Ule), + ("<=", CmpOp::Le), + ("=s", CmpOp::Sge), + (">=u", CmpOp::Uge), + (">=", CmpOp::Ge), + (">s", CmpOp::Sgt), + (">u", CmpOp::Ugt), + (">", CmpOp::Gt), +]; /// Parse a function from its text form. The returned function owns its memory. pub fn parse(text: &str) -> Result { @@ -132,7 +226,7 @@ fn all_digits(s: &[u8]) -> bool { !s.is_empty() && s.iter().all(|&c| is_digit(c)) } -/// BinOp from its printer word (`add`, `sub`, `bit_and`, ...). The text is untrusted, +/// BinOp from its word (`add`, `sub`, `bit_and`, `udiv`, ...). The text is untrusted, /// so an unknown op word is a parse error, never an invalid enum. fn bin_op_of(word: &str) -> Option { match word { @@ -141,12 +235,32 @@ fn bin_op_of(word: &str) -> Option { "mul" => Some(BinOp::Mul), "div" => Some(BinOp::Div), "rem" => Some(BinOp::Rem), + "udiv" => Some(BinOp::UDiv), + "sdiv" => Some(BinOp::SDiv), + "urem" => Some(BinOp::URem), + "srem" => Some(BinOp::SRem), "bit_and" => Some(BinOp::BitAnd), "bit_or" => Some(BinOp::BitOr), "bit_xor" => Some(BinOp::BitXor), "shl" => Some(BinOp::Shl), "shr" => Some(BinOp::Shr), - "mulh" => Some(BinOp::Mulh), + "sar" => Some(BinOp::Sar), + "umulh" => Some(BinOp::UMulh), + "smulh" => Some(BinOp::SMulh), + _ => None, + } +} + +fn convert_kind_of(word: &str) -> Option { + match word { + "trunc" => Some(ConvertKind::Trunc), + "zext" => Some(ConvertKind::Zext), + "sext" => Some(ConvertKind::Sext), + "sitofp" => Some(ConvertKind::SiToFp), + "uitofp" => Some(ConvertKind::UiToFp), + "fptosi" => Some(ConvertKind::FpToSi), + "fptoui" => Some(ConvertKind::FpToUi), + "fpresize" => Some(ConvertKind::FpResize), _ => None, } } @@ -209,8 +323,10 @@ fn barrier_scope_of(word: &str) -> Option { fn atomic_op_of(word: &str) -> Option { match word { "add" => Some(AtomicOp::Add), - "min" => Some(AtomicOp::Min), - "max" => Some(AtomicOp::Max), + "smin" => Some(AtomicOp::SMin), + "smax" => Some(AtomicOp::SMax), + "umin" => Some(AtomicOp::UMin), + "umax" => Some(AtomicOp::UMax), "bit_and" => Some(AtomicOp::BitAnd), "bit_or" => Some(AtomicOp::BitOr), "bit_xor" => Some(AtomicOp::BitXor), @@ -542,20 +658,6 @@ impl<'a> FunctionParser<'a> { let align = u32::try_from(n).map_err(|_| ParseError::InvalidSyntax)?; return Ok(Attribute::Align(align)); } - if word == "endian" { - self.eat(b'(')?; - let e = self.read_word().to_string(); - self.eat(b')')?; - // The text is UNTRUSTED, so an unknown order is a parse error and never an - // invalid enum. - let order = match e.as_str() { - "little" => Endianness::Little, - "big" => Endianness::Big, - "native" => Endianness::Native, - _ => return Err(ParseError::InvalidSyntax), - }; - return Ok(Attribute::Endian(order)); - } // Namespaced: `namespace.key` with an optional `= value`. A namespace may itself // hold dots (`vulcan.gpu` is the one the whole GPU path uses), so read the full // dotted path and split it at the LAST dot: everything before it is the @@ -715,6 +817,13 @@ impl<'a> FunctionParser<'a> { return Err(ParseError::InvalidSyntax); } self.parse_void_call(block)?; + } else if word == "intrinsic" { + if !pending.is_empty() { + return Err(ParseError::InvalidSyntax); + } + let (name, args, imm, side_effects) = self.parse_intrinsic_body()?; + self.func + .append_intrinsic_void(block, &name, &args, imm, side_effects); } else if word == "call_indirect" { if !pending.is_empty() { return Err(ParseError::InvalidSyntax); @@ -889,36 +998,24 @@ impl<'a> FunctionParser<'a> { .map_err(|_| ParseError::InvalidSyntax) } + /// Consume the binary arithmetic operator at the cursor, if there is one. + fn read_arith_op(&mut self) -> Option { + let rest = &self.src[self.pos..]; + let (text, op) = ARITH_OPS + .iter() + .find(|(text, _)| rest.starts_with(text.as_bytes()))?; + self.pos += text.len(); + Some(*op) + } + fn read_cmp_op(&mut self) -> Result { - match self.peek().ok_or(ParseError::InvalidSyntax)? { - b'=' => { - self.eat(b'=')?; - self.eat(b'=')?; - Ok(CmpOp::Eq) - } - b'!' => { - self.eat(b'!')?; - self.eat(b'=')?; - Ok(CmpOp::Ne) - } - b'<' => { - self.pos += 1; - Ok(if self.try_char(b'=') { - CmpOp::Le - } else { - CmpOp::Lt - }) - } - b'>' => { - self.pos += 1; - Ok(if self.try_char(b'=') { - CmpOp::Ge - } else { - CmpOp::Gt - }) - } - _ => Err(ParseError::InvalidSyntax), - } + let rest = &self.src[self.pos..]; + let (text, op) = CMP_OPS + .iter() + .find(|(text, _)| rest.starts_with(text.as_bytes())) + .ok_or(ParseError::InvalidSyntax)?; + self.pos += text.len(); + Ok(*op) } fn parse_let(&mut self, block: Block) -> Result { @@ -953,17 +1050,7 @@ impl<'a> FunctionParser<'a> { return Ok(result); } self.skip_ws(); - let rest = &self.src[self.pos..]; - if rest.starts_with(b"<<") { - self.pos += 2; - return self.finish_arith(block, lhs, BinOp::Shl); - } - if rest.starts_with(b">>") { - self.pos += 2; - return self.finish_arith(block, lhs, BinOp::Shr); - } - if let Some(bop) = arith_op_of(self.peek().unwrap_or(0)) { - self.pos += 1; + if let Some(bop) = self.read_arith_op() { return self.finish_arith(block, lhs, bop); } let op = self.read_cmp_op()?; @@ -978,21 +1065,24 @@ impl<'a> FunctionParser<'a> { } let op = self.read_word().to_string(); if op == "load" { - let is_volatile = self.try_word("volatile"); + let mem = self.parse_mem_flags()?; self.skip_ws(); let ty = self.parse_type()?; self.skip_ws(); self.eat(b',')?; self.skip_ws(); let ptr = self.parse_value_ref()?; - let result = self.func.append_inst( - block, - ty, - Opcode::Load(Load { - ptr, - volatile: is_volatile, - }), - ); + let result = self.func.append_load(block, ty, ptr, mem); + self.record_value(result); + return Ok(result); + } + if op == "intrinsic" { + self.skip_ws(); + let ty = self.parse_type()?; + let (name, args, imm, side_effects) = self.parse_intrinsic_body()?; + let result = self + .func + .append_intrinsic(block, ty, &name, &args, imm, side_effects); self.record_value(result); return Ok(result); } @@ -1018,6 +1108,12 @@ impl<'a> FunctionParser<'a> { return Ok(result); } if op == "dot" { + self.skip_ws(); + let signed = match self.read_word() { + "signed" => true, + "unsigned" => false, + _ => return Err(ParseError::InvalidSyntax), + }; self.skip_ws(); let acc = self.parse_value_ref()?; self.skip_ws(); @@ -1028,7 +1124,7 @@ impl<'a> FunctionParser<'a> { self.eat(b',')?; self.skip_ws(); let b = self.parse_value_ref()?; - let result = self.func.append_dot(block, acc, a, b); + let result = self.func.append_dot(block, acc, a, b, signed); self.record_value(result); return Ok(result); } @@ -1149,15 +1245,15 @@ impl<'a> FunctionParser<'a> { return Ok(result); } if op == "convert" { + self.skip_ws(); + let kind = convert_kind_of(self.read_word()).ok_or(ParseError::InvalidSyntax)?; self.skip_ws(); let ty = self.parse_type()?; self.skip_ws(); self.eat(b',')?; self.skip_ws(); let value = self.parse_value_ref()?; - let result = self - .func - .append_inst(block, ty, Opcode::Convert(Convert { value })); + let result = self.func.append_convert(block, ty, kind, value); self.record_value(result); return Ok(result); } @@ -1173,7 +1269,6 @@ impl<'a> FunctionParser<'a> { self.func.types.intern(TypeKind::Float(FloatKind::F32)) } else { self.func.types.intern(TypeKind::Int(IntDesc { - signed: false, bits: format.payload_bits(), })) }; @@ -1208,10 +1303,7 @@ impl<'a> FunctionParser<'a> { let ty = if dequantize { self.func.types.intern(TypeKind::Float(FloatKind::F32)) } else { - self.func.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits: 8, - })) + self.func.types.intern(TypeKind::Int(IntDesc { bits: 8 })) }; let conversion = NvFp4Convert { value: operands[0], @@ -1290,18 +1382,107 @@ impl<'a> FunctionParser<'a> { }) } + /// Read `be` or `le` at the cursor; any other word leaves the cursor where it was. + fn try_endian_word(&mut self) -> Option { + let save = self.pos; + self.skip_ws(); + match self.read_word() { + "be" => Some(Endianness::Big), + "le" => Some(Endianness::Little), + _ => { + self.pos = save; + None + } + } + } + + /// Parse the optional modifier words after `load`/`store`: `volatile`, `atomic `, + /// `align `, `be`, `le`, in any order, each at most once. Absent modifiers keep the + /// `MemFlags::new()` default (native byte order, not atomic, no alignment promise). + fn parse_mem_flags(&mut self) -> Result { + let mut mem = MemFlags::new(); + let (mut volatile, mut atomic, mut align, mut endian) = (false, false, false, false); + loop { + if self.try_word("volatile") { + if core::mem::replace(&mut volatile, true) { + return Err(ParseError::InvalidSyntax); + } + mem.volatile = true; + } else if self.try_word("atomic") { + if core::mem::replace(&mut atomic, true) { + return Err(ParseError::InvalidSyntax); + } + self.skip_ws(); + mem.ordering = + Some(atomic_ordering_of(self.read_word()).ok_or(ParseError::InvalidSyntax)?); + } else if self.try_word("align") { + if core::mem::replace(&mut align, true) { + return Err(ParseError::InvalidSyntax); + } + self.skip_ws(); + mem.align = + u32::try_from(self.read_unsigned()?).map_err(|_| ParseError::InvalidSyntax)?; + } else if let Some(order) = self.try_endian_word() { + if core::mem::replace(&mut endian, true) { + return Err(ParseError::InvalidSyntax); + } + mem.endian = order; + } else { + return Ok(mem); + } + } + } + fn parse_store(&mut self, block: Block) -> Result<(), ParseError> { - let is_volatile = self.try_word("volatile"); + let mem = self.parse_mem_flags()?; self.skip_ws(); let value = self.parse_value_ref()?; self.skip_ws(); self.eat(b',')?; self.skip_ws(); let ptr = self.parse_value_ref()?; - self.func.append_store_vol(block, value, ptr, is_volatile); + self.func.append_store_mem(block, value, ptr, mem); Ok(()) } + /// The part of an intrinsic after its optional result type: + /// `@name(args) [imm N] [pure]`. The name is a dotted identifier (`aarch64.aese`); + /// `pure` clears `side_effects`, which is otherwise set. + fn parse_intrinsic_body(&mut self) -> Result<(String, Vec, i64, bool), ParseError> { + self.skip_ws(); + self.eat(b'@')?; + let start = self.pos; + while matches!(self.src.get(self.pos), Some(&c) if is_word_char(c) || c == b'.') { + self.pos += 1; + } + if self.pos == start { + return Err(ParseError::InvalidSyntax); + } + let name = core::str::from_utf8(&self.src[start..self.pos]) + .map_err(|_| ParseError::InvalidSyntax)? + .to_string(); + let mut args: Vec = Vec::new(); + self.parse_call_args(&mut args)?; + let (mut imm, mut side_effects) = (0, true); + let (mut saw_imm, mut saw_pure) = (false, false); + loop { + if self.try_word("imm") { + if core::mem::replace(&mut saw_imm, true) { + return Err(ParseError::InvalidSyntax); + } + self.skip_ws(); + imm = self.read_signed()?; + } else if self.try_word("pure") { + if core::mem::replace(&mut saw_pure, true) { + return Err(ParseError::InvalidSyntax); + } + side_effects = false; + } else { + return Ok((name, args, imm, side_effects)); + } + } + } + fn parse_barrier(&mut self, block: Block) -> Result<(), ParseError> { self.skip_ws(); let scope = barrier_scope_of(self.read_word()).ok_or(ParseError::InvalidSyntax)?; @@ -1839,11 +2020,472 @@ mod tests { #[test] fn round_trips_the_canonical_max_function() { let func = parse_ok( - "fn {\n block0(v0: i32, v1: i32):\n let v2 = v0 > v1\n if v2 { block1(v0) } else { block1(v1) }\n ret void\n\n block1(v3: i32):\n ret v3\n}", + "fn {\n block0(v0: i32, v1: i32):\n let v2 = v0 >s v1\n if v2 { block1(v0) } else { block1(v1) }\n ret void\n\n block1(v3: i32):\n ret v3\n}", ); let diags = verify::verify(&func, verify::Profile::High); assert!(diags.ok()); let diags = verify::verify(&func, verify::Profile::Low); assert!(diags.ok()); } + + fn only_mem(func: &Function, which: usize) -> MemFlags { + let mut seen = 0; + for bi in 0..func.block_count() { + for &inst in func.block_insts(Block::from_u32(bi as u32)) { + let mem = match func.opcode(inst) { + Opcode::Load(l) => l.mem, + Opcode::Store(st) => st.mem, + _ => continue, + }; + if seen == which { + return mem; + } + seen += 1; + } + } + panic!("no memory access #{which}"); + } + + /// The canonical spelling of `mem` as documented in the module comment. + fn mem_words(mem: &MemFlags) -> String { + let mut out = String::new(); + if mem.volatile { + out.push_str("volatile "); + } + if let Some(o) = mem.ordering { + out.push_str(match o { + AtomicOrdering::Relaxed => "atomic relaxed ", + AtomicOrdering::Acquire => "atomic acquire ", + AtomicOrdering::Release => "atomic release ", + AtomicOrdering::AcqRel => "atomic acq_rel ", + AtomicOrdering::SeqCst => "atomic seq_cst ", + }); + } + if mem.align != 0 { + out.push_str(&alloc::format!("align {} ", mem.align)); + } + match mem.endian { + Endianness::Native => {} + Endianness::Big => out.push_str("be "), + Endianness::Little => out.push_str("le "), + } + out + } + + #[test] + fn parses_load_and_store_without_modifiers_as_plain() { + let func = parse_ok( + "fn {\n block0(v0: ptr, v1: i32):\n let v2 = load i32, v0\n store v1, v0\n ret v2\n}", + ); + assert_eq!(only_mem(&func, 0), MemFlags::new()); + assert_eq!(only_mem(&func, 1), MemFlags::new()); + assert!(only_mem(&func, 0).is_plain()); + } + + #[test] + fn mem_flags_text_round_trips_every_modifier() { + let orderings = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Acquire), + Some(AtomicOrdering::Release), + Some(AtomicOrdering::AcqRel), + Some(AtomicOrdering::SeqCst), + ]; + let endians = [Endianness::Native, Endianness::Little, Endianness::Big]; + for volatile in [false, true] { + for align in [0u32, 1, 8, 4096] { + for ordering in orderings { + for endian in endians { + let mem = MemFlags { + volatile, + align, + ordering, + endian, + }; + let words = mem_words(&mem); + let text = alloc::format!( + "fn {{\n block0(v0: ptr, v1: i32):\n let v2 = load {words}i32, v0\n store {words}v1, v0\n ret v2\n}}" + ); + let func = parse_ok(&text); + assert_eq!(only_mem(&func, 0), mem, "{text}"); + assert_eq!(only_mem(&func, 1), mem, "{text}"); + } + } + } + } + } + + #[test] + fn mem_flag_modifiers_parse_in_any_order() { + let func = parse_ok( + "fn {\n block0(v0: ptr):\n let v1 = load be align 8 atomic seq_cst volatile i64, v0\n ret v1\n}", + ); + assert_eq!( + only_mem(&func, 0), + MemFlags { + volatile: true, + align: 8, + ordering: Some(AtomicOrdering::SeqCst), + endian: Endianness::Big, + } + ); + } + + #[test] + fn rejects_malformed_mem_flags() { + for words in [ + "volatile volatile ", + "align ", + "align x ", + "atomic ", + "atomic strong ", + "atomic acquire atomic acquire ", + "be le ", + "align 4 align 4 ", + ] { + let text = alloc::format!( + "fn {{\n block0(v0: ptr):\n let v1 = load {words}i32, v0\n ret v1\n}}" + ); + assert!(parse(&text).is_err(), "{text}"); + } + } + + #[test] + fn round_trips_a_valued_intrinsic() { + let func = parse_ok( + "fn {\n block0(v0: i32, v1: i32):\n let v2 = intrinsic i32 @aarch64.aese(v0, v1) imm -3\n ret v2\n}", + ); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + let Opcode::Intrinsic(i) = func.opcode(inst) else { + panic!("expected intrinsic"); + }; + assert_eq!(func.symbol_name(i.symbol), "aarch64.aese"); + assert_eq!( + func.value_list(i.args), + &[Value::from_u32(0), Value::from_u32(1)] + ); + assert_eq!(i.imm, -3); + assert!(i.side_effects); + } + + #[test] + fn round_trips_a_pure_and_a_void_intrinsic() { + let func = parse_ok( + "fn {\n block0(v0: i32):\n let v1 = intrinsic i32 @x86.bswap(v0) pure\n intrinsic @arm.dmb() imm 15\n intrinsic @nop.pure() pure imm 1\n ret v1\n}", + ); + let insts = func.block_insts(Block::from_u32(0)); + let get = |n: usize| match func.opcode(insts[n]) { + Opcode::Intrinsic(i) => i, + _ => panic!("expected intrinsic"), + }; + let (a, b, c) = (get(0), get(1), get(2)); + assert!(!a.side_effects && a.imm == 0); + assert!(func.inst_result(insts[1]).is_none()); + assert!(b.side_effects && b.imm == 15 && b.args.len == 0); + assert!(!c.side_effects && c.imm == 1); + assert_eq!(func.symbol_name(b.symbol), "arm.dmb"); + } + + // --- Signedness on the operation --------------------------------------------- + + const ALL_BIN_OPS: [BinOp; 17] = [ + BinOp::Add, + BinOp::Sub, + BinOp::Mul, + BinOp::Div, + BinOp::Rem, + BinOp::BitAnd, + BinOp::BitOr, + BinOp::BitXor, + BinOp::Shl, + BinOp::Shr, + BinOp::Sar, + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::UMulh, + BinOp::SMulh, + ]; + + const ALL_CMP_OPS: [CmpOp; 14] = [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Lt, + CmpOp::Le, + CmpOp::Gt, + CmpOp::Ge, + CmpOp::Slt, + CmpOp::Sle, + CmpOp::Sgt, + CmpOp::Sge, + CmpOp::Ult, + CmpOp::Ule, + CmpOp::Ugt, + CmpOp::Uge, + ]; + + fn assert_verifies(func: &Function, what: &str) { + let diags = verify::verify(func, verify::Profile::High); + assert!(diags.ok(), "{what}: {:?}", diags.items()); + } + + fn type_text(func: &Function, v: u32) -> String { + func.types + .display(func.value_type(Value::from_u32(v))) + .to_string() + } + + #[test] + fn every_binary_operator_parses_from_its_own_symbol() { + for op in ALL_BIN_OPS { + let ty = if matches!(op, BinOp::Div | BinOp::Rem) { + "f32" + } else { + "i32" + }; + let text = alloc::format!( + "fn {{\n block0(v0: {ty}, v1: {ty}):\n let v2 = v0 {} v1\n ret v2\n}}", + op.symbol() + ); + let func = parse_ok(&text); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::Arith(a) + if a.op == op && a.lhs == Value::from_u32(0) && a.rhs == Value::from_u32(1)), + "{op:?}: {text}" + ); + assert_verifies(&func, &text); + } + } + + #[test] + fn every_binary_operator_has_an_immediate_form() { + for op in ALL_BIN_OPS { + let ty = if matches!(op, BinOp::Div | BinOp::Rem) { + "f32" + } else { + "i32" + }; + let text = alloc::format!( + "fn {{\n block0(v0: {ty}):\n let v1 = v0 {} 3\n ret v1\n}}", + op.symbol() + ); + let func = parse_ok(&text); + let inst = func.defining_inst(Value::from_u32(1)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::ArithImm(a) if a.op == op && a.imm == 3), + "{op:?}: {text}" + ); + assert_verifies(&func, &text); + } + } + + #[test] + fn operator_suffixes_pick_the_signed_or_unsigned_variant() { + let op_of = |sym: &str| { + let func = parse_ok(&alloc::format!( + "fn {{\n block0(v0: i64, v1: i64):\n let v2 = v0 {sym} v1\n ret v2\n}}" + )); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + match func.opcode(inst) { + Opcode::Arith(a) => a.op, + other => panic!("expected arith, got {other:?}"), + } + }; + assert_eq!(op_of(">>"), BinOp::Shr); + assert_eq!(op_of(">>s"), BinOp::Sar); + assert_eq!(op_of("/u"), BinOp::UDiv); + assert_eq!(op_of("/s"), BinOp::SDiv); + assert_eq!(op_of("%u"), BinOp::URem); + assert_eq!(op_of("%s"), BinOp::SRem); + assert_eq!(op_of("*"), BinOp::Mul); + assert_eq!(op_of("*hu"), BinOp::UMulh); + assert_eq!(op_of("*hs"), BinOp::SMulh); + } + + #[test] + fn every_comparison_parses_from_its_own_symbol() { + for op in ALL_CMP_OPS { + let ty = if matches!(op, CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge) { + "f32" + } else { + "i32" + }; + let text = alloc::format!( + "fn {{\n block0(v0: {ty}, v1: {ty}):\n let v2 = v0 {} v1\n ret v2\n}}", + op.symbol() + ); + let func = parse_ok(&text); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::Icmp(c) if c.op == op), + "{op:?}: {text}" + ); + assert_eq!(type_text(&func, 2), "bool"); + assert_verifies(&func, &text); + } + let ptrs = + parse_ok("fn {\n block0(v0: ptr, v1: ptr):\n let v2 = v0 >"), + ("f32", ">>s"), + ("f32", "*hu"), + ("i32", "<"), + ("i32", ">="), + ("f32", "u"), + ("ptr", "", "<4 x i32>"), + ] { + let text = alloc::format!( + "fn {{\n block0(v0: {src}):\n let v1 = convert {} {dst}, v0\n ret v1\n}}", + kind.name() + ); + let func = parse_ok(&text); + let inst = func.defining_inst(Value::from_u32(1)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::Convert(c) + if c.kind == kind && c.value == Value::from_u32(0)), + "{text}" + ); + assert_eq!(type_text(&func, 1), dst, "{text}"); + assert_verifies(&func, &text); + } + } + + #[test] + fn a_conversion_needs_a_known_kind() { + for text in [ + "fn {\n block0(v0: i8):\n let v1 = convert i32, v0\n ret v1\n}", + "fn {\n block0(v0: i8):\n let v1 = convert widen i32, v0\n ret v1\n}", + "fn {\n block0(v0: i8):\n let v1 = convert sext, v0\n ret v1\n}", + ] { + assert!(parse(text).is_err(), "{text}"); + } + } + + #[test] + fn dot_spells_its_signedness() { + for (word, signed) in [("signed", true), ("unsigned", false)] { + let func = parse_ok(&alloc::format!( + "fn {{\n block0(v0: <4 x i32>, v1: <16 x i8>, v2: <16 x i8>):\n let v3 = dot {word} v0, v1, v2\n ret v3\n}}" + )); + let inst = func.defining_inst(Value::from_u32(3)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::Dot(d) + if d.signed == signed && d.acc == Value::from_u32(0) + && d.a == Value::from_u32(1) && d.b == Value::from_u32(2)), + "{word}" + ); + assert_eq!(type_text(&func, 3), "<4 x i32>"); + } + for text in [ + "fn {\n block0(v0: <4 x i32>, v1: <16 x i8>, v2: <16 x i8>):\n let v3 = dot v0, v1, v2\n ret v3\n}", + "fn {\n block0(v0: <4 x i32>, v1: <16 x i8>, v2: <16 x i8>):\n let v3 = dot sgn v0, v1, v2\n ret v3\n}", + ] { + assert!(parse(text).is_err(), "{text}"); + } + } + + #[test] + fn atomic_min_and_max_are_signed_or_unsigned() { + for (word, op) in [ + ("add", AtomicOp::Add), + ("smin", AtomicOp::SMin), + ("smax", AtomicOp::SMax), + ("umin", AtomicOp::UMin), + ("umax", AtomicOp::UMax), + ("bit_and", AtomicOp::BitAnd), + ("bit_or", AtomicOp::BitOr), + ("bit_xor", AtomicOp::BitXor), + ("exchange", AtomicOp::Exchange), + ] { + let func = parse_ok(&alloc::format!( + "fn {{\n block0(v0: ptr, v1: i32):\n let v2 = atomic_rmw {word} device seq_cst v0, v1\n ret v2\n}}" + )); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::AtomicRmw(a) if a.op == op), + "{word}" + ); + assert_verifies(&func, word); + } + for word in ["min", "max"] { + let text = alloc::format!( + "fn {{\n block0(v0: ptr, v1: i32):\n let v2 = atomic_rmw {word} device seq_cst v0, v1\n ret v2\n}}" + ); + assert!(parse(&text).is_err(), "{word}"); + } + } + + #[test] + fn integer_types_have_no_unsigned_spelling_and_mulh_has_no_bare_word() { + assert!(parse("fn {\n block0(v0: u32):\n ret v0\n}").is_err()); + assert!(parse("fn {\n block0():\n const v0: u8 = 1\n ret v0\n}").is_err()); + assert!(parse("fn {\n block0():\n const v0: i8 = 1\n ret v0\n}").is_ok()); + for word in ["mulh", "min", "max"] { + let text = alloc::format!( + "fn {{\n block0(v0: <4 x i32>):\n let v1 = reduce {word}, v0\n ret v1\n}}" + ); + assert!(parse(&text).is_err(), "{word}"); + } + for word in ["umulh", "sdiv", "sar"] { + let text = alloc::format!( + "fn {{\n block0(v0: <4 x i32>):\n let v1 = reduce {word}, v0\n ret v1\n}}" + ); + // The word parses; the verifier is what limits reductions to the lane-wise ops. + let func = parse_ok(&text); + assert!(!verify::verify(&func, verify::Profile::High).ok(), "{word}"); + } + } + + #[test] + fn low_float_and_nvfp4_results_are_plain_integers() { + let func = parse_ok( + "fn {\n block0(v0: f32, v1: i8, v2: f32):\n let v3 = encode_low_float bf16, v0\n let v4 = encode_low_float f8_e4m3, v0\n let v5 = quantize_nvfp4 multiply, divide, v0, v1, v2\n ret v3\n}", + ); + assert_eq!(type_text(&func, 3), "i16"); + assert_eq!(type_text(&func, 4), "i8"); + assert_eq!(type_text(&func, 5), "i8"); + } } diff --git a/src/ir/src/softfp.rs b/src/ir/src/softfp.rs index ec6e3dd..54443ed 100644 --- a/src/ir/src/softfp.rs +++ b/src/ir/src/softfp.rs @@ -2,7 +2,7 @@ extern crate alloc; use alloc::vec::Vec; -use super::function::{BinOp, Block, CmpOp, Function, Inst, Opcode, Value}; +use super::function::{BinOp, Block, CmpOp, Convert, ConvertKind, Function, Inst, Opcode, Value}; use super::types::Type; use super::types::{FloatKind, IntDesc, TypeKind}; @@ -52,17 +52,20 @@ fn float_kind(func: &Function, v: Value) -> Option { } } -struct IntInfo { - bits: u16, - signed: bool, +/// The width of the integer `v` has, if it is one. +fn int_bits(func: &Function, v: Value) -> Option { + match func.types.type_kind(func.value_type(v)) { + TypeKind::Int(i) => Some(i.bits), + _ => None, + } } -fn int_info(func: &Function, v: Value) -> Option { - match func.types.type_kind(func.value_type(v)) { - TypeKind::Int(i) => Some(IntInfo { - bits: i.bits, - signed: i.signed, - }), +/// The width of the integer a soft-float routine takes or returns for a `bits`-wide one: `int` +/// (32) up to 32 bits and `long long` (64) up to 64 bits; nothing wider has a routine. +fn routine_bits(bits: u16) -> Option { + match bits { + 1..=32 => Some(32), + 33..=64 => Some(64), _ => None, } } @@ -77,22 +80,22 @@ fn arith_sym(op: BinOp) -> Option<&'static str> { } } -fn cmp_sym(op: CmpOp) -> &'static str { +/// The comparison routine of a float compare and the SIGNED integer compare of its `int` +/// result against zero that gives the compare's value (the routines return `< 0`, `0`, `> 0`). +fn cmp_sym(op: CmpOp) -> Option<(&'static str, CmpOp)> { match op { - CmpOp::Eq => "__eqtf2", - CmpOp::Ne => "__netf2", - CmpOp::Lt => "__lttf2", - CmpOp::Le => "__letf2", - CmpOp::Gt => "__gttf2", - CmpOp::Ge => "__getf2", + CmpOp::Eq => Some(("__eqtf2", CmpOp::Eq)), + CmpOp::Ne => Some(("__netf2", CmpOp::Ne)), + CmpOp::Lt => Some(("__lttf2", CmpOp::Slt)), + CmpOp::Le => Some(("__letf2", CmpOp::Sle)), + CmpOp::Gt => Some(("__gttf2", CmpOp::Sgt)), + CmpOp::Ge => Some(("__getf2", CmpOp::Sge)), + _ => None, } } pub fn lower(func: &mut Function) -> bool { - let i32_ty = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_ty = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let f128_ty = func.types.intern(TypeKind::Float(FloatKind::F128)); let mut changed = false; for bi in 0..func.block_count() { @@ -171,7 +174,10 @@ fn lower_inst( if !is_f128(func, c.lhs) { return false; } - let status = emit_call(func, order, i32_ty, cmp_sym(c.op), &[c.lhs, c.rhs]); + let Some((routine, against_zero)) = cmp_sym(c.op) else { + return false; + }; + let status = emit_call(func, order, i32_ty, routine, &[c.lhs, c.rhs]); let zero = emit_inst(func, order, i32_ty, Opcode::Iconst(0)); let res_ty = func.value_type(result); let cmp = emit_inst( @@ -179,7 +185,7 @@ fn lower_inst( order, res_ty, Opcode::Icmp(super::function::Compare { - op: c.op, + op: against_zero, lhs: status, rhs: zero, }), @@ -188,21 +194,94 @@ fn lower_inst( true } Opcode::Convert(cv) => { - let src_q = is_f128(func, cv.value); - let dst_q = is_f128(func, result); - if !src_q && !dst_q { - return false; - } - if src_q && dst_q { + let (src_q, dst_q) = (is_f128(func, cv.value), is_f128(func, result)); + // Neither side is f128, or both are (not a conversion): nothing to call. + if src_q == dst_q { return false; } let res_ty = func.value_type(result); - let sym = if dst_q { - to_f128_sym(func, cv.value) - } else { - from_f128_sym(func, cv.value, res_ty, func) + let call = match (cv.kind, dst_q) { + (ConvertKind::FpResize, true) => { + let routine = match float_kind(func, cv.value) { + Some(FloatKind::F32) => "__extendsftf2", + Some(FloatKind::F64) => "__extenddftf2", + Some(FloatKind::F16) => "__extendhftf2", + _ => return false, + }; + emit_call(func, order, res_ty, routine, &[cv.value]) + } + (ConvertKind::SiToFp | ConvertKind::UiToFp, true) => { + // The routines take an `int` / `long long`: a narrower source is widened + // first, sign- or zero-extending as the conversion reads it. + let signed = cv.kind == ConvertKind::SiToFp; + let Some(bits) = int_bits(func, cv.value) else { + return false; + }; + let Some(arg_bits) = routine_bits(bits) else { + return false; + }; + let arg_ty = func.types.intern(TypeKind::Int(IntDesc { bits: arg_bits })); + let arg = if bits < arg_bits { + let kind = if signed { + ConvertKind::Sext + } else { + ConvertKind::Zext + }; + let widen = Opcode::Convert(Convert { + value: cv.value, + kind, + }); + emit_inst(func, order, arg_ty, widen) + } else { + cv.value + }; + let routine = match (signed, arg_bits) { + (true, 32) => "__floatsitf", + (true, _) => "__floatditf", + (false, 32) => "__floatunsitf", + (false, _) => "__floatunditf", + }; + emit_call(func, order, res_ty, routine, &[arg]) + } + (ConvertKind::FpResize, false) => { + let routine = match float_kind(func, result) { + Some(FloatKind::F32) => "__trunctfsf2", + Some(FloatKind::F64) => "__trunctfdf2", + Some(FloatKind::F16) => "__trunctfhf2", + _ => return false, + }; + emit_call(func, order, res_ty, routine, &[cv.value]) + } + (ConvertKind::FpToSi | ConvertKind::FpToUi, false) => { + // The routine returns an `int` / `long long`; a narrower destination is + // that result truncated. + let signed = cv.kind == ConvertKind::FpToSi; + let Some(bits) = int_bits(func, result) else { + return false; + }; + let Some(ret_bits) = routine_bits(bits) else { + return false; + }; + let ret_ty = func.types.intern(TypeKind::Int(IntDesc { bits: ret_bits })); + let routine = match (signed, ret_bits) { + (true, 32) => "__fixtfsi", + (true, _) => "__fixtfdi", + (false, 32) => "__fixunstfsi", + (false, _) => "__fixunstfdi", + }; + let raw = emit_call(func, order, ret_ty, routine, &[cv.value]); + if bits < ret_bits { + let narrow = Opcode::Convert(Convert { + value: raw, + kind: ConvertKind::Trunc, + }); + emit_inst(func, order, res_ty, narrow) + } else { + raw + } + } + _ => return false, }; - let call = emit_call(func, order, res_ty, sym, &[cv.value]); func.replace_all_uses(result, call); true } @@ -218,48 +297,203 @@ fn lower_inst( } } -fn to_f128_sym(func: &Function, src: Value) -> &'static str { - if let Some(fk) = float_kind(func, src) { - return match fk { - FloatKind::F32 => "__extendsftf2", - FloatKind::F64 => "__extenddftf2", - FloatKind::F16 => "__extendhftf2", - FloatKind::F128 => unreachable!(), +#[cfg(test)] +mod tests { + use alloc::format; + use alloc::string::String; + use alloc::vec; + + use super::*; + use crate::function::{Compare, Ret, Terminator}; + use crate::verify::{Profile, verify}; + + /// One word per instruction of the entry block: the routine of a call, the kind of a + /// convert, the operator of a compare. + fn shape(func: &Function) -> Vec { + func.block_insts(Block::from_u32(0)) + .iter() + .map(|&inst| match func.opcode(inst) { + Opcode::Call(c) => format!("call {}", func.symbol_name(c.symbol)), + Opcode::Convert(c) => format!("convert {}", c.kind.name()), + Opcode::Icmp(c) => format!("icmp {}", c.op.symbol()), + Opcode::Iconst(_) => String::from("iconst"), + Opcode::Arith(a) => format!("arith {}", a.op.symbol()), + other => panic!("unexpected {other:?}"), + }) + .collect() + } + + /// `fn(x: src) -> dst { ret convert dst, x }` after `lower`. + fn lowered_convert(kind: ConvertKind, src: &str, dst: &str) -> (Function, bool) { + let mut func = Function::new(); + let (s, d) = ( + func.types.parse_type(src).unwrap(), + func.types.parse_type(dst).unwrap(), + ); + let entry = func.append_block(); + let x = func.append_block_param(entry, s); + let r = func.append_convert(entry, d, kind, x); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + let changed = lower(&mut func); + (func, changed) + } + + fn check(kind: ConvertKind, src: &str, dst: &str, want: &[&str]) { + let (func, changed) = lowered_convert(kind, src, dst); + assert!(changed, "{kind:?} {src} -> {dst}"); + assert_eq!(shape(&func), want, "{kind:?} {src} -> {dst}"); + let diags = verify(&func, Profile::High); + assert!(diags.ok(), "{kind:?} {src} -> {dst}: {:?}", diags.items()); + // The result the function returns is the last instruction's. + let Some(Terminator::Ret(ret)) = func.terminator(Block::from_u32(0)) else { + panic!("ret"); }; + let last = *func.block_insts(Block::from_u32(0)).last().unwrap(); + assert_eq!(func.inst_result(last), Some(ret.values[0])); + assert_eq!(func.value_type(ret.values[0]), { + let mut types = func.types.clone(); + types.parse_type(dst).unwrap() + }); + } + + #[test] + fn integers_narrower_than_the_routine_are_widened_as_the_conversion_reads_them() { + use ConvertKind::*; + check(SiToFp, "i8", "f128", &["convert sext", "call __floatsitf"]); + check(SiToFp, "i16", "f128", &["convert sext", "call __floatsitf"]); + check(SiToFp, "i32", "f128", &["call __floatsitf"]); + check(SiToFp, "i40", "f128", &["convert sext", "call __floatditf"]); + check(SiToFp, "i64", "f128", &["call __floatditf"]); + check( + UiToFp, + "i8", + "f128", + &["convert zext", "call __floatunsitf"], + ); + check(UiToFp, "i32", "f128", &["call __floatunsitf"]); + check( + UiToFp, + "i40", + "f128", + &["convert zext", "call __floatunditf"], + ); + check(UiToFp, "i64", "f128", &["call __floatunditf"]); + } + + #[test] + fn f128_to_integer_calls_the_routine_of_the_destination_sign_and_truncates() { + use ConvertKind::*; + check(FpToSi, "f128", "i32", &["call __fixtfsi"]); + check(FpToSi, "f128", "i8", &["call __fixtfsi", "convert trunc"]); + check(FpToSi, "f128", "i64", &["call __fixtfdi"]); + check(FpToSi, "f128", "i48", &["call __fixtfdi", "convert trunc"]); + check(FpToUi, "f128", "i32", &["call __fixunstfsi"]); + check( + FpToUi, + "f128", + "i16", + &["call __fixunstfsi", "convert trunc"], + ); + check(FpToUi, "f128", "i64", &["call __fixunstfdi"]); + } + + #[test] + fn f128_float_resizes_call_the_extend_and_truncate_routines() { + use ConvertKind::*; + check(FpResize, "f32", "f128", &["call __extendsftf2"]); + check(FpResize, "f64", "f128", &["call __extenddftf2"]); + check(FpResize, "f16", "f128", &["call __extendhftf2"]); + check(FpResize, "f128", "f32", &["call __trunctfsf2"]); + check(FpResize, "f128", "f64", &["call __trunctfdf2"]); + check(FpResize, "f128", "f16", &["call __trunctfhf2"]); } - let ii = int_info(func, src).expect("softfp convert source is a number"); - if ii.signed { - if ii.bits <= 32 { - "__floatsitf" - } else { - "__floatditf" + + #[test] + fn conversions_with_no_f128_or_no_routine_are_left_alone() { + use ConvertKind::*; + for (kind, src, dst) in [ + (SiToFp, "i32", "f32"), + (FpToUi, "f64", "i64"), + (FpResize, "f32", "f64"), + (Zext, "i8", "i32"), + // Wider than any routine's integer. + (SiToFp, "i128", "f128"), + (FpToSi, "f128", "i128"), + // Kind and types disagree (the verifier rejects these). + (Zext, "i8", "f128"), + (FpResize, "i32", "f128"), + ] { + let (func, changed) = lowered_convert(kind, src, dst); + assert!(!changed, "{kind:?} {src} -> {dst}"); + assert_eq!(shape(&func).len(), 1); } - } else if ii.bits <= 32 { - "__floatunsitf" - } else { - "__floatunditf" } -} -fn from_f128_sym(func: &Function, src: Value, dst_ty: Type, _f: &Function) -> &'static str { - let _ = src; - match func.types.type_kind(dst_ty) { - TypeKind::Float(FloatKind::F32) => "__trunctfsf2", - TypeKind::Float(FloatKind::F64) => "__trunctfdf2", - TypeKind::Float(FloatKind::F16) => "__trunctfhf2", - TypeKind::Int(i) => { - if i.signed { - if i.bits <= 32 { - "__fixtfsi" - } else { - "__fixtfdi" - } - } else if i.bits <= 32 { - "__fixunstfsi" - } else { - "__fixunstfdi" - } + #[test] + fn f128_comparisons_test_the_routines_int_result_against_zero_signed() { + for (op, routine, against_zero) in [ + (CmpOp::Eq, "__eqtf2", CmpOp::Eq), + (CmpOp::Ne, "__netf2", CmpOp::Ne), + (CmpOp::Lt, "__lttf2", CmpOp::Slt), + (CmpOp::Le, "__letf2", CmpOp::Sle), + (CmpOp::Gt, "__gttf2", CmpOp::Sgt), + (CmpOp::Ge, "__getf2", CmpOp::Sge), + ] { + let mut func = Function::new(); + let f128 = func.types.parse_type("f128").unwrap(); + let bool_t = func.types.parse_type("bool").unwrap(); + let entry = func.append_block(); + let a = func.append_block_param(entry, f128); + let b = func.append_block_param(entry, f128); + let r = func.append_inst(entry, bool_t, Opcode::Icmp(Compare { op, lhs: a, rhs: b })); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert!(lower(&mut func)); + assert_eq!( + shape(&func), + vec![ + format!("call {routine}"), + String::from("iconst"), + format!("icmp {}", against_zero.symbol()), + ] + ); + let diags = verify(&func, Profile::High); + assert!(diags.ok(), "{op:?}: {:?}", diags.items()); } - _ => unreachable!(), + } + + #[test] + fn f128_arithmetic_and_non_f128_instructions() { + let mut func = Function::new(); + let f128 = func.types.parse_type("f128").unwrap(); + let i32_t = func.types.parse_type("i32").unwrap(); + let entry = func.append_block(); + let a = func.append_block_param(entry, f128); + let b = func.append_block_param(entry, f128); + let x = func.append_block_param(entry, i32_t); + let y = func.append_block_param(entry, i32_t); + // Integer division is not an f128 operation and must not be touched. + let q = func.append_inst( + entry, + i32_t, + Opcode::Arith(crate::function::Arith { + op: BinOp::SDiv, + lhs: x, + rhs: y, + }), + ); + let s = func.append_inst( + entry, + f128, + Opcode::Arith(crate::function::Arith { + op: BinOp::Div, + lhs: a, + rhs: b, + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::many(&[q, s]))); + assert!(lower(&mut func)); + assert_eq!(shape(&func), ["arith /s", "call __divtf3"]); + let diags = verify(&func, Profile::High); + assert!(diags.ok(), "{:?}", diags.items()); } } diff --git a/src/ir/src/types.rs b/src/ir/src/types.rs index e8855b0..4cfe8db 100644 --- a/src/ir/src/types.rs +++ b/src/ir/src/types.rs @@ -7,12 +7,6 @@ use core::fmt; #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub struct Type(pub u32); -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] -pub enum Signedness { - Signed, - Unsigned, -} - #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum FloatKind { F32, @@ -23,7 +17,6 @@ pub enum FloatKind { #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub struct IntDesc { - pub signed: bool, pub bits: u16, } @@ -102,6 +95,20 @@ impl TypeTable { &self.kinds[h.0 as usize] } + /// Size in bytes of an int/float/ptr scalar as it sits in memory (pointers are 8 bytes); + /// None for bool, vectors, aggregates and integers whose width is not a whole byte count. + pub fn scalar_mem_bytes(&self, h: Type) -> Option { + match self.type_kind(h) { + TypeKind::Int(d) if d.bits % 8 == 0 && d.bits != 0 => Some(u32::from(d.bits) / 8), + TypeKind::Float(FloatKind::F16) => Some(2), + TypeKind::Float(FloatKind::F32) => Some(4), + TypeKind::Float(FloatKind::F64) => Some(8), + TypeKind::Float(FloatKind::F128) => Some(16), + TypeKind::Ptr(_) => Some(8), + _ => None, + } + } + pub fn count(&self) -> usize { self.kinds.len() } @@ -230,12 +237,9 @@ impl TypeTable { "f128" => Ok(self.intern(TypeKind::Float(FloatKind::F128))), _ => { let b = w.as_bytes(); - if b.len() >= 2 && (b[0] == b'i' || b[0] == b'u') { + if b.len() >= 2 && b[0] == b'i' { let bits: u16 = w[1..].parse().map_err(|_| ParseError)?; - Ok(self.intern(TypeKind::Int(IntDesc { - signed: b[0] == b'i', - bits, - }))) + Ok(self.intern(TypeKind::Int(IntDesc { bits }))) } else { Err(ParseError) } @@ -249,7 +253,7 @@ impl TypeTable { match &self.kinds[ty.0 as usize] { TypeKind::Bool => out.push_str("bool"), TypeKind::Int(i) => { - out.push(if i.signed { 'i' } else { 'u' }); + out.push('i'); out.push_str(&alloc::format!("{}", i.bits)); } TypeKind::Float(f) => out.push_str(match f { @@ -354,16 +358,24 @@ mod tests { #[test] fn scalars_round_trip() { let mut t = TypeTable::new(); - let i32a = t.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32a = t.intern(TypeKind::Int(IntDesc { bits: 32 })); assert_eq!(i32a, t.parse_type("i32").unwrap()); assert_eq!(t.parse_type("bool").unwrap(), t.intern(TypeKind::Bool)); assert_eq!(t.ptr_global(), t.parse_type("ptr").unwrap()); assert_eq!("i32", alloc::format!("{}", t.display(i32a))); } + #[test] + fn integer_types_are_signless_and_the_u_spelling_is_gone() { + let mut t = TypeTable::new(); + let i8t = t.parse_type("i8").unwrap(); + assert_eq!(i8t, t.intern(TypeKind::Int(IntDesc { bits: 8 }))); + assert_eq!(i8t, t.parse_type(" i8 ").unwrap()); + for text in ["u8", "u32", "u64", "i", "i-1", "i8x", "i99999"] { + assert!(t.parse_type(text).is_err(), "{text}"); + } + } + #[test] fn dedup_and_spaces() { let mut t = TypeTable::new(); @@ -378,10 +390,7 @@ mod tests { #[test] fn composites() { let mut t = TypeTable::new(); - let i32t = t.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32t = t.intern(TypeKind::Int(IntDesc { bits: 32 })); let f32t = t.intern(TypeKind::Float(FloatKind::F32)); let v4 = t.intern(TypeKind::Vector(VectorDesc { len: 4, elem: i32t })); assert_eq!(v4, t.parse_type("<4 x i32>").unwrap()); diff --git a/src/ir/src/verify.rs b/src/ir/src/verify.rs index 38b97fb..b373c89 100644 --- a/src/ir/src/verify.rs +++ b/src/ir/src/verify.rs @@ -2,8 +2,11 @@ extern crate alloc; use alloc::vec::Vec; -use super::function::{AttrTarget, Function, Opcode, Value}; -use super::types::TypeKind; +use super::attribute::Endianness; +use super::function::{ + AtomicOrdering, BinOp, CmpOp, ConvertKind, Function, Inst, MemFlags, Opcode, Value, +}; +use super::types::{Type, TypeKind}; #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Profile { @@ -29,8 +32,12 @@ pub enum Diagnostic { value: Value, block: super::function::Block, }, - MisplacedEndian(AttrTarget), + InvalidMemFlags(super::function::Inst), OperandTypeMismatch(Value), + /// The operator is not defined for its operand (or result) types: integer division on + /// floats, a signed compare on pointers, an ill-formed conversion, and so on. Carries + /// the offending instruction. + InvalidOperation(Inst), } #[derive(Debug, Default)] @@ -56,8 +63,9 @@ pub fn verify(func: &Function, profile: Profile) -> Diagnostics { let mut diags = Diagnostics::default(); check_edges(func, &mut diags); check_dominance(func, &mut diags); - check_attributes(func, &mut diags); + check_mem_flags(func, &mut diags); check_operand_types(func, &mut diags); + check_operations(func, &mut diags); if profile == Profile::Low { check_primitive_types(func, &mut diags); } @@ -295,6 +303,7 @@ fn opcode_operands(func: &Function, inst: super::function::Inst) -> Vec { out.extend_from_slice(func.value_list(cf.then.args)); out.extend_from_slice(func.value_list(cf.else_.args)); } + Opcode::Intrinsic(i) => out.extend_from_slice(func.value_list(i.args)), Opcode::Iconst(_) | Opcode::Fconst(_) | Opcode::Fconst128(_) @@ -305,30 +314,62 @@ fn opcode_operands(func: &Function, inst: super::function::Inst) -> Vec { out } -fn is_memory_op(op: &Opcode) -> bool { - matches!( - op, - Opcode::Load(_) | Opcode::Store(_) | Opcode::Prefetch(_) | Opcode::Matmul(_) - ) -} - -fn check_attributes(func: &Function, diags: &mut Diagnostics) { - for entry in func.attribute_entries() { - if let super::attribute::Attribute::Endian(_) = entry.attr { - let ok = match entry.target { - AttrTarget::Value(v) => func - .defining_inst(v) - .is_some_and(|i| is_memory_op(&func.opcode(i))), - AttrTarget::Inst(i) => is_memory_op(&func.opcode(i)), - AttrTarget::Func | AttrTarget::Block(_) => false, +/// Checks the `MemFlags` of every load and store against the accessed type. +fn check_mem_flags(func: &Function, diags: &mut Diagnostics) { + for bi in 0..func.block_count() { + let b = super::function::Block::from_u32(bi as u32); + for &inst in func.block_insts(b) { + let (mem, ty, is_store) = match func.opcode_ref(inst) { + Opcode::Load(l) => match func.inst_result(inst) { + Some(r) => (l.mem, func.value_type(r), false), + None => { + diags.list.push(Diagnostic::InvalidMemFlags(inst)); + continue; + } + }, + Opcode::Store(st) => (st.mem, func.value_type(st.value), true), + _ => continue, }; - if !ok { - diags.list.push(Diagnostic::MisplacedEndian(entry.target)); + if !mem_flags_valid(func, &mem, ty, is_store) { + diags.list.push(Diagnostic::InvalidMemFlags(inst)); } } } } +/// Whether `mem` is a legal flag set for a load (`is_store == false`) or store of `ty`. +pub(crate) fn mem_flags_valid(func: &Function, mem: &MemFlags, ty: Type, is_store: bool) -> bool { + if mem.align != 0 && !mem.align.is_power_of_two() { + return false; + } + let bytes = func.types.scalar_mem_bytes(ty); + if let Some(ordering) = mem.ordering { + let allowed = if is_store { + matches!( + ordering, + AtomicOrdering::Relaxed | AtomicOrdering::Release | AtomicOrdering::SeqCst + ) + } else { + matches!( + ordering, + AtomicOrdering::Relaxed | AtomicOrdering::Acquire | AtomicOrdering::SeqCst + ) + }; + let sized = bytes.is_some_and(|n| n.is_power_of_two() && n <= 8); + if !allowed || !sized { + return false; + } + } + if mem.endian != Endianness::Little && mem.endian != Endianness::Native { + // A byte swap only has a meaning on a 1, 2, 4 or 8 byte int/float/ptr scalar + // (one byte is a no-op). + if !bytes.is_some_and(|n| matches!(n, 1 | 2 | 4 | 8)) { + return false; + } + } + true +} + fn is_ptr(func: &Function, v: Value) -> bool { matches!(func.types.type_kind(func.value_type(v)), TypeKind::Ptr(_)) } @@ -412,6 +453,158 @@ fn check_operand_types(func: &Function, diags: &mut Diagnostics) { } } +/// The kind of a scalar type, or of the lanes of a vector type. +fn lane_kind(func: &Function, ty: Type) -> &TypeKind { + match func.types.type_kind(ty) { + TypeKind::Vector(v) => func.types.type_kind(v.elem), + other => other, + } +} + +/// The lane count of a vector type; `None` for a scalar. +fn lane_count(func: &Function, ty: Type) -> Option { + match func.types.type_kind(ty) { + TypeKind::Vector(v) => Some(v.len), + _ => None, + } +} + +/// The width in bits of an integer (or `bool`, one bit) lane. +fn lane_bits(kind: &TypeKind) -> Option { + match kind { + TypeKind::Int(i) => Some(i.bits), + TypeKind::Bool => Some(1), + _ => None, + } +} + +fn lane_is_int(kind: &TypeKind) -> bool { + matches!(kind, TypeKind::Int(_)) +} + +fn lane_is_float(kind: &TypeKind) -> bool { + matches!(kind, TypeKind::Float(_)) +} + +/// Whether `op` is defined for operands whose lanes are `lane`. +fn binop_accepts(op: BinOp, lane: &TypeKind) -> bool { + match op { + BinOp::Add | BinOp::Sub => { + lane_is_int(lane) || lane_is_float(lane) || matches!(lane, TypeKind::Ptr(_)) + } + BinOp::Mul => lane_is_int(lane) || lane_is_float(lane), + BinOp::Div | BinOp::Rem => lane_is_float(lane), + BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => { + lane_is_int(lane) || matches!(lane, TypeKind::Bool) + } + BinOp::Shl + | BinOp::Shr + | BinOp::Sar + | BinOp::UDiv + | BinOp::SDiv + | BinOp::URem + | BinOp::SRem + | BinOp::UMulh + | BinOp::SMulh => lane_is_int(lane), + } +} + +/// Whether `op` is defined for operands whose lanes are `lane`. +fn cmpop_accepts(op: CmpOp, lane: &TypeKind) -> bool { + match op { + CmpOp::Eq | CmpOp::Ne => matches!( + lane, + TypeKind::Bool | TypeKind::Int(_) | TypeKind::Float(_) | TypeKind::Ptr(_) + ), + CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge => lane_is_float(lane), + CmpOp::Slt | CmpOp::Sle | CmpOp::Sgt | CmpOp::Sge => lane_is_int(lane), + CmpOp::Ult | CmpOp::Ule | CmpOp::Ugt | CmpOp::Uge => { + lane_is_int(lane) || matches!(lane, TypeKind::Ptr(_)) + } + } +} + +/// Whether converting `src` to `dst` is the conversion `kind` (see [`ConvertKind`]). +fn convert_ok(func: &Function, kind: ConvertKind, src: Type, dst: Type) -> bool { + if lane_count(func, src) != lane_count(func, dst) { + return false; + } + let (s, d) = (lane_kind(func, src), lane_kind(func, dst)); + match kind { + ConvertKind::Trunc => { + lane_is_int(s) + && lane_is_int(d) + && lane_bits(d).unwrap_or(0) < lane_bits(s).unwrap_or(0) + } + ConvertKind::Zext => { + lane_bits(s).is_some() + && lane_is_int(d) + && lane_bits(d).unwrap_or(0) > lane_bits(s).unwrap_or(0) + } + ConvertKind::Sext => { + lane_is_int(s) + && lane_is_int(d) + && lane_bits(d).unwrap_or(0) > lane_bits(s).unwrap_or(0) + } + ConvertKind::SiToFp | ConvertKind::UiToFp => lane_is_int(s) && lane_is_float(d), + ConvertKind::FpToSi | ConvertKind::FpToUi => lane_is_float(s) && lane_is_int(d), + ConvertKind::FpResize => lane_is_float(s) && lane_is_float(d) && s != d, + } +} + +/// Whether the arithmetic `op` over `lhs` (and `rhs`) producing `result` is well typed. +/// Pointer arithmetic (`ptr +/- int`, `int + ptr`) mixes types, so only the operator rule is +/// checked for it; otherwise the result has the operand type. +fn arith_ok( + func: &Function, + op: BinOp, + lhs: Value, + rhs: Option, + result: Option, +) -> bool { + let lt = func.value_type(lhs); + let lane = lane_kind(func, lt); + if !binop_accepts(op, lane) { + return false; + } + let mixed = rhs.is_some_and(|r| func.value_type(r) != lt); + if mixed || matches!(lane, TypeKind::Ptr(_)) { + return true; + } + result.is_none_or(|r| func.value_type(r) == lt) +} + +/// Checks that every operator is defined for the types it is applied to: the signed / +/// unsigned / floating-point operator families (`BinOp`, `CmpOp`) and the legal type pairs +/// of each `ConvertKind`. +fn check_operations(func: &Function, diags: &mut Diagnostics) { + for bi in 0..func.block_count() { + let b = super::function::Block::from_u32(bi as u32); + for &inst in func.block_insts(b) { + let result = func.inst_result(inst); + let ok = match func.opcode_ref(inst) { + Opcode::Arith(a) => arith_ok(func, a.op, a.lhs, Some(a.rhs), result), + Opcode::ArithImm(a) => arith_ok(func, a.op, a.lhs, None, result), + Opcode::Icmp(c) => { + let lt = func.value_type(c.lhs); + let scalar_result = result.is_none_or(|r| { + lane_count(func, lt).is_some() + || matches!(func.types.type_kind(func.value_type(r)), TypeKind::Bool) + }); + cmpop_accepts(c.op, lane_kind(func, lt)) && scalar_result + } + Opcode::Convert(cv) => result.is_some_and(|r| { + convert_ok(func, cv.kind, func.value_type(cv.value), func.value_type(r)) + }), + _ => true, + }; + if !ok { + diags.list.push(Diagnostic::InvalidOperation(inst)); + } + } + } +} + fn pointer_arith(func: &Function, lhs: Value, rhs: Value, op: super::function::BinOp) -> bool { if !matches!( op, @@ -439,3 +632,572 @@ fn check_primitive_types(func: &Function, diags: &mut Diagnostics) { } } } + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + use crate::function::{Arith, Compare, Ret, Terminator}; + use crate::types::{FloatKind, IntDesc, VectorDesc}; + + fn mem(ordering: Option, endian: Endianness, align: u32) -> MemFlags { + MemFlags { + volatile: false, + align, + ordering, + endian, + } + } + + fn int(bits: u16) -> TypeKind { + TypeKind::Int(IntDesc { bits }) + } + + /// Number of `InvalidMemFlags` diagnostics for a load and for a store of `kind` with `m`. + fn bad_counts(kind: TypeKind, m: MemFlags) -> (usize, usize) { + let count = |is_store: bool| { + let mut func = Function::new(); + let ty = func.types.intern(kind.clone()); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + if is_store { + let v = func.append_block_param(entry, ty); + func.append_store_mem(entry, v, p, m); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + } else { + let v = func.append_load(entry, ty, p, m); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + } + let diags = verify(&func, Profile::High); + assert!( + diags + .items() + .iter() + .all(|d| matches!(d, Diagnostic::InvalidMemFlags(_))), + "{:?}", + diags.items() + ); + diags.count() + }; + (count(false), count(true)) + } + + fn ok(kind: TypeKind, m: MemFlags) -> bool { + bad_counts(kind, m) == (0, 0) + } + + const NATIVE: Endianness = Endianness::Native; + + #[test] + fn default_and_plain_flags_verify() { + assert!(ok(int(32), MemFlags::new())); + assert!(ok(TypeKind::Bool, MemFlags::new())); + assert!(ok( + TypeKind::Struct(vec![]), + mem(None, Endianness::Little, 0) + )); + let volatile = MemFlags { + volatile: true, + ..MemFlags::new() + }; + assert!(ok(int(8), volatile)); + } + + #[test] + fn rejects_an_ordering_invalid_for_the_access_kind() { + use AtomicOrdering::*; + for o in [Relaxed, Acquire, SeqCst] { + let (load, _) = bad_counts(int(32), mem(Some(o), NATIVE, 0)); + assert_eq!(load, 0, "load {o:?}"); + } + for o in [Release, AcqRel] { + let (load, _) = bad_counts(int(32), mem(Some(o), NATIVE, 0)); + assert_eq!(load, 1, "load {o:?}"); + } + for o in [Relaxed, Release, SeqCst] { + let (_, store) = bad_counts(int(32), mem(Some(o), NATIVE, 0)); + assert_eq!(store, 0, "store {o:?}"); + } + for o in [Acquire, AcqRel] { + let (_, store) = bad_counts(int(32), mem(Some(o), NATIVE, 0)); + assert_eq!(store, 1, "store {o:?}"); + } + } + + #[test] + fn atomic_accesses_need_a_power_of_two_scalar_of_at_most_8_bytes() { + let seq = Some(AtomicOrdering::SeqCst); + for bits in [8, 16, 32, 64] { + assert!(ok(int(bits), mem(seq, NATIVE, 0)), "i{bits}"); + } + assert!(ok(TypeKind::Float(FloatKind::F32), mem(seq, NATIVE, 0))); + assert!(ok(TypeKind::Float(FloatKind::F64), mem(seq, NATIVE, 0))); + assert!(ok(TypeKind::Float(FloatKind::F16), mem(seq, NATIVE, 0))); + assert!(ok( + TypeKind::Ptr(crate::types::AddressSpace::Global), + mem(seq, NATIVE, 0) + )); + assert_eq!(bad_counts(int(128), mem(seq, NATIVE, 0)), (1, 1), "i128"); + assert_eq!(bad_counts(int(24), mem(seq, NATIVE, 0)), (1, 1), "i24"); + assert_eq!(bad_counts(int(1), mem(seq, NATIVE, 0)), (1, 1), "i1"); + assert_eq!(bad_counts(TypeKind::Bool, mem(seq, NATIVE, 0)), (1, 1)); + assert_eq!( + bad_counts(TypeKind::Float(FloatKind::F128), mem(seq, NATIVE, 0)), + (1, 1) + ); + assert_eq!( + bad_counts(TypeKind::Struct(vec![]), mem(seq, NATIVE, 0)), + (1, 1) + ); + } + + #[test] + fn big_endian_needs_a_1_2_4_or_8_byte_scalar() { + let big = Endianness::Big; + for bits in [8, 16, 32, 64] { + assert!(ok(int(bits), mem(None, big, 0)), "i{bits}"); + } + assert!(ok(TypeKind::Float(FloatKind::F32), mem(None, big, 0))); + assert!(ok(TypeKind::Float(FloatKind::F16), mem(None, big, 0))); + assert!(ok( + TypeKind::Ptr(crate::types::AddressSpace::Shared), + mem(None, big, 0) + )); + assert_eq!(bad_counts(int(128), mem(None, big, 0)), (1, 1)); + assert_eq!(bad_counts(int(24), mem(None, big, 0)), (1, 1)); + assert_eq!(bad_counts(TypeKind::Bool, mem(None, big, 0)), (1, 1)); + assert_eq!( + bad_counts(TypeKind::Float(FloatKind::F128), mem(None, big, 0)), + (1, 1) + ); + assert_eq!( + bad_counts(TypeKind::Struct(vec![]), mem(None, big, 0)), + (1, 1) + ); + let mut func = Function::new(); + let elem = func.types.intern(int(32)); + let vector = TypeKind::Vector(VectorDesc { len: 4, elem }); + assert_eq!(bad_counts(vector.clone(), mem(None, big, 0)), (1, 1)); + assert!(ok(vector, mem(None, Endianness::Little, 0))); + } + + #[test] + fn rejects_an_align_that_is_not_a_power_of_two() { + for align in [0, 1, 2, 16, 4096] { + assert!(ok(int(32), mem(None, NATIVE, align)), "align {align}"); + } + for align in [3, 6, 12, 4095] { + assert_eq!( + bad_counts(int(32), mem(None, NATIVE, align)), + (1, 1), + "align {align}" + ); + } + } + + #[test] + fn a_load_with_no_result_is_rejected() { + let mut func = Function::new(); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + func.append_stmt_raw( + entry, + Opcode::Load(crate::function::Load { + ptr: p, + mem: MemFlags::new(), + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + assert_eq!(verify(&func, Profile::High).count(), 1); + } + + #[test] + fn intrinsic_operands_are_dominance_checked() { + let mut func = Function::new(); + let i32_t = func.types.intern(int(32)); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let r = func.append_intrinsic(entry, i32_t, "t.op", &[a], 1, false); + func.append_intrinsic_void(entry, "t.sink", &[r, a], 0, true); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert!(verify(&func, Profile::High).ok()); + } + + // --- Operator / type rules --------------------------------------------------- + + /// The type shapes the operator tables are checked against; vectors have 4 lanes. + #[derive(Clone, Copy, Debug)] + enum K { + Bool, + I(u16), + F(FloatKind), + Ptr, + VI(u16), + VF(FloatKind), + } + + fn make(func: &mut Function, k: K) -> Type { + let scalar = |func: &mut Function, kind: TypeKind| func.types.intern(kind); + match k { + K::Bool => scalar(func, TypeKind::Bool), + K::I(bits) => scalar(func, int(bits)), + K::F(f) => scalar(func, TypeKind::Float(f)), + K::Ptr => func.types.ptr_global(), + K::VI(bits) => { + let elem = scalar(func, int(bits)); + scalar(func, TypeKind::Vector(VectorDesc { len: 4, elem })) + } + K::VF(f) => { + let elem = scalar(func, TypeKind::Float(f)); + scalar(func, TypeKind::Vector(VectorDesc { len: 4, elem })) + } + } + } + + /// Asserts every diagnostic is an `InvalidOperation` and returns how many there are. + fn invalid_operations(func: &Function) -> usize { + let diags = verify(func, Profile::High); + assert!( + diags + .items() + .iter() + .all(|d| matches!(d, Diagnostic::InvalidOperation(_))), + "{:?}", + diags.items() + ); + diags.count() + } + + /// `fn(a: k, b: k) { ret a b }`. + fn arith_bad(k: K, op: BinOp) -> usize { + let mut func = Function::new(); + let ty = make(&mut func, k); + let entry = func.append_block(); + let a = func.append_block_param(entry, ty); + let b = func.append_block_param(entry, ty); + let r = func.append_inst(entry, ty, Opcode::Arith(Arith { op, lhs: a, rhs: b })); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + invalid_operations(&func) + } + + /// `fn(a: k) { ret a 1 }`. + fn arith_imm_bad(k: K, op: BinOp) -> usize { + let mut func = Function::new(); + let ty = make(&mut func, k); + let entry = func.append_block(); + let a = func.append_block_param(entry, ty); + let r = func.append_arith_imm(entry, ty, op, a, 1); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + invalid_operations(&func) + } + + /// `fn(a: k, b: k) { ret a b }` for a comparison; the result is `bool` (a `bool` + /// vector for vector operands). + fn cmp_bad(k: K, op: CmpOp) -> usize { + let mut func = Function::new(); + let ty = make(&mut func, k); + let bool_t = func.types.intern(TypeKind::Bool); + let result_t = match k { + K::VI(_) | K::VF(_) => func.types.intern(TypeKind::Vector(VectorDesc { + len: 4, + elem: bool_t, + })), + _ => bool_t, + }; + let entry = func.append_block(); + let a = func.append_block_param(entry, ty); + let b = func.append_block_param(entry, ty); + let r = func.append_inst( + entry, + result_t, + Opcode::Icmp(Compare { op, lhs: a, rhs: b }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + invalid_operations(&func) + } + + /// `fn(a: src) { ret convert dst, a }`. + fn convert_bad(kind: ConvertKind, src: K, dst: K) -> usize { + let mut func = Function::new(); + let (s, d) = (make(&mut func, src), make(&mut func, dst)); + let entry = func.append_block(); + let a = func.append_block_param(entry, s); + let r = func.append_convert(entry, d, kind, a); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + invalid_operations(&func) + } + + const F32: K = K::F(FloatKind::F32); + const F64: K = K::F(FloatKind::F64); + + #[test] + fn arithmetic_operators_are_split_by_integer_and_float_domain() { + use BinOp::*; + // (operator, defined on integers, defined on floats) + let table = [ + (Add, true, true), + (Sub, true, true), + (Mul, true, true), + (Div, false, true), + (Rem, false, true), + (UDiv, true, false), + (SDiv, true, false), + (URem, true, false), + (SRem, true, false), + (Shl, true, false), + (Shr, true, false), + (Sar, true, false), + (UMulh, true, false), + (SMulh, true, false), + (BitAnd, true, false), + (BitOr, true, false), + (BitXor, true, false), + ]; + for (op, on_int, on_float) in table { + for k in [K::I(8), K::I(32), K::I(64), K::VI(32)] { + assert_eq!(arith_bad(k, op), usize::from(!on_int), "{op:?} on {k:?}"); + assert_eq!( + arith_imm_bad(k, op), + usize::from(!on_int), + "{op:?} imm {k:?}" + ); + } + for k in [F32, F64, K::F(FloatKind::F16), K::VF(FloatKind::F32)] { + assert_eq!(arith_bad(k, op), usize::from(!on_float), "{op:?} on {k:?}"); + assert_eq!( + arith_imm_bad(k, op), + usize::from(!on_float), + "{op:?} imm {k:?}" + ); + } + } + } + + #[test] + fn bitwise_operators_also_accept_bool_but_nothing_else_does() { + for op in [BinOp::BitAnd, BinOp::BitOr, BinOp::BitXor] { + assert_eq!(arith_bad(K::Bool, op), 0, "{op:?}"); + } + for op in [BinOp::Add, BinOp::Mul, BinOp::Shl, BinOp::UDiv, BinOp::Div] { + assert_eq!(arith_bad(K::Bool, op), 1, "{op:?} on bool"); + } + for op in [BinOp::Mul, BinOp::Shr, BinOp::SDiv, BinOp::BitAnd] { + assert_eq!(arith_bad(K::Ptr, op), 1, "{op:?} on ptr"); + } + } + + #[test] + fn pointer_arithmetic_still_mixes_pointer_and_integer_operands() { + let mut func = Function::new(); + let ptr_t = func.types.ptr_global(); + let i64_t = func.types.intern(int(64)); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let n = func.append_block_param(entry, i64_t); + let q = func.append_inst( + entry, + ptr_t, + Opcode::Arith(Arith { + op: BinOp::Add, + lhs: p, + rhs: n, + }), + ); + let r = func.append_arith_imm(entry, ptr_t, BinOp::Sub, q, 8); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert!(verify(&func, Profile::High).ok()); + } + + #[test] + fn an_arithmetic_result_must_have_the_operand_type() { + let mut func = Function::new(); + let i32_t = func.types.intern(int(32)); + let i64_t = func.types.intern(int(64)); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let b = func.append_block_param(entry, i32_t); + let r = func.append_inst( + entry, + i64_t, + Opcode::Arith(Arith { + op: BinOp::Add, + lhs: a, + rhs: b, + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert_eq!(invalid_operations(&func), 1); + } + + #[test] + fn comparison_operators_are_split_by_domain() { + use CmpOp::*; + // (operator, ints, floats, bools, pointers) + let table = [ + (Eq, true, true, true, true), + (Ne, true, true, true, true), + (Lt, false, true, false, false), + (Le, false, true, false, false), + (Gt, false, true, false, false), + (Ge, false, true, false, false), + (Slt, true, false, false, false), + (Sle, true, false, false, false), + (Sgt, true, false, false, false), + (Sge, true, false, false, false), + (Ult, true, false, false, true), + (Ule, true, false, false, true), + (Ugt, true, false, false, true), + (Uge, true, false, false, true), + ]; + for (op, on_int, on_float, on_bool, on_ptr) in table { + for k in [K::I(8), K::I(32), K::I(64), K::VI(32)] { + assert_eq!(cmp_bad(k, op), usize::from(!on_int), "{op:?} on {k:?}"); + } + for k in [F32, F64, K::VF(FloatKind::F32)] { + assert_eq!(cmp_bad(k, op), usize::from(!on_float), "{op:?} on {k:?}"); + } + assert_eq!( + cmp_bad(K::Bool, op), + usize::from(!on_bool), + "{op:?} on bool" + ); + assert_eq!(cmp_bad(K::Ptr, op), usize::from(!on_ptr), "{op:?} on ptr"); + } + } + + #[test] + fn a_scalar_comparison_yields_bool() { + let mut func = Function::new(); + let i32_t = func.types.intern(int(32)); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let b = func.append_block_param(entry, i32_t); + let r = func.append_inst( + entry, + i32_t, + Opcode::Icmp(Compare { + op: CmpOp::Ult, + lhs: a, + rhs: b, + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert_eq!(invalid_operations(&func), 1); + } + + #[test] + fn conversions_accept_exactly_their_type_pairs() { + use ConvertKind::*; + let (i8, i16, i32, i64) = (K::I(8), K::I(16), K::I(32), K::I(64)); + let f16 = K::F(FloatKind::F16); + let f128 = K::F(FloatKind::F128); + let good: &[(ConvertKind, K, K)] = &[ + (Trunc, i32, i8), + (Trunc, i64, i32), + (Trunc, K::VI(32), K::VI(16)), + (Zext, K::Bool, i8), + (Zext, K::Bool, i32), + (Zext, i8, i32), + (Zext, i32, i64), + (Zext, K::VI(8), K::VI(32)), + (Sext, i8, i16), + (Sext, i32, i64), + (Sext, K::VI(16), K::VI(32)), + (SiToFp, i32, F32), + (SiToFp, i64, F64), + (SiToFp, K::VI(32), K::VF(FloatKind::F32)), + (UiToFp, i8, F32), + (UiToFp, i64, f128), + (FpToSi, F32, i32), + (FpToSi, F64, i8), + (FpToSi, K::VF(FloatKind::F32), K::VI(32)), + (FpToUi, F32, i32), + (FpToUi, f128, i64), + (FpResize, F32, F64), + (FpResize, F64, F32), + (FpResize, f16, f128), + (FpResize, K::VF(FloatKind::F32), K::VF(FloatKind::F64)), + ]; + for &(kind, src, dst) in good { + assert_eq!( + convert_bad(kind, src, dst), + 0, + "{kind:?} {src:?} -> {dst:?}" + ); + } + let bad: &[(ConvertKind, K, K)] = &[ + // Trunc must narrow, and between ints. + (Trunc, i8, i32), + (Trunc, i32, i32), + (Trunc, F64, F32), + (Trunc, i32, K::Bool), + // Zext / Sext must widen; Sext does not take bool. + (Zext, i32, i8), + (Zext, i32, i32), + (Zext, F32, i64), + (Sext, i32, i8), + (Sext, i32, i32), + (Sext, K::Bool, i32), + (Sext, F32, F64), + // Int -> float and float -> int only. + (SiToFp, F32, F64), + (SiToFp, K::Bool, F32), + (UiToFp, F32, i32), + (FpToSi, i32, F32), + (FpToUi, i32, i64), + (FpToUi, F32, F64), + // FpResize needs two different float types. + (FpResize, F32, F32), + (FpResize, i32, F32), + (FpResize, F32, i32), + // Lane counts must agree. + (Zext, i8, K::VI(32)), + (SiToFp, K::VI(32), F32), + (FpResize, F32, K::VF(FloatKind::F64)), + ]; + for &(kind, src, dst) in bad { + assert_eq!( + convert_bad(kind, src, dst), + 1, + "{kind:?} {src:?} -> {dst:?}" + ); + } + } + + #[test] + fn pointers_are_never_converted_pointer_integer_casts_are_reinterprets() { + for kind in [ + ConvertKind::Trunc, + ConvertKind::Zext, + ConvertKind::Sext, + ConvertKind::SiToFp, + ConvertKind::UiToFp, + ConvertKind::FpToSi, + ConvertKind::FpToUi, + ConvertKind::FpResize, + ] { + assert_eq!(convert_bad(kind, K::Ptr, K::I(64)), 1, "{kind:?}"); + assert_eq!(convert_bad(kind, K::I(32), K::Ptr), 1, "{kind:?}"); + } + } + + #[test] + fn a_convert_without_a_result_is_rejected() { + let mut func = Function::new(); + let i32_t = func.types.intern(int(32)); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + func.append_stmt_raw( + entry, + Opcode::Convert(crate::function::Convert { + value: a, + kind: ConvertKind::Zext, + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + assert_eq!(invalid_operations(&func), 1); + } +} diff --git a/src/emulator/src/aarch64/decode.rs b/src/target/src/aarch64/decode.rs similarity index 94% rename from src/emulator/src/aarch64/decode.rs rename to src/target/src/aarch64/decode.rs index 8611d7b..65ac6e6 100644 --- a/src/emulator/src/aarch64/decode.rs +++ b/src/target/src/aarch64/decode.rs @@ -307,6 +307,28 @@ pub struct Address { pub rd: u8, pub offset: i64, } +/// The operands of a `SYS`-space system instruction (`DC`, `IC`, `TLBI`, `AT`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SysOp { + pub op1: u8, + pub crn: u8, + pub crm: u8, + pub op2: u8, + pub rt: u8, +} +/// `PRFM`/`PRFUM` with a register-based address. `op` is the prefetch operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Prefetch { + pub op: u8, + pub rn: u8, + pub addressing: Addressing, +} +/// `PRFM` (literal): a prefetch from a PC-relative address. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PrefetchLiteral { + pub op: u8, + pub offset: i64, +} /// How a modified-immediate instruction combines its immediate with `rd`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ImmCombine { @@ -834,18 +856,23 @@ pub enum Instruction { SimdLiteral(SimdLiteral), Exclusive(Exclusive), Atomic(Atomic), - Clrex, + Clrex(u8), Adr(Address), System(System), DcZva(u8), AddressTranslate(AddressTranslate), Eret, - CacheOp, - Tlbi, + CacheOp(SysOp), + Tlbi(SysOp), Nop, - Dsb, - Dmb, - Isb, + /// A hint-space `HINT #n` other than `nop` and the named hints (`bti`, `esb`, + /// `csdb`, ...). Executes as `nop`. + Hint(u8), + Prefetch(Prefetch), + PrefetchLiteral(PrefetchLiteral), + Dsb(u8), + Dmb(u8), + Isb(u8), Sev, Sevl, Wfe, @@ -877,15 +904,13 @@ pub enum Instruction { q: bool, }, SimdDupElement(SimdDupElement), - /// An instruction the host executes from its raw word (see `host`). - Host(u32), SimdInsert(SimdInsert), SimdInsertElement(SimdInsertElement), SimdExtract(SimdExtract), SimdExt(SimdExt), SimdAlu(SimdAlu), SimdTable(SimdTable), - Svc, + Svc(u16), Psci, Wfi, } @@ -926,17 +951,16 @@ impl Instruction { | TestBranch(_) | Call(_) | Indirect(_) - | Svc + | Svc(_) | Psci | Wfi | Wfe | Brk(_) - | Isb + | Isb(_) | DcZva(_) | AddressTranslate(_) - | Host(_) | Eret - | Tlbi + | Tlbi(_) ) } } @@ -1357,13 +1381,22 @@ fn system_register( }) } +/// The 32- or 64-bit pattern of the 8-bit floating-point immediate `imm8` +/// (`VFPExpandImm`), as `fmov` and the vector `fmov` take it. +pub fn expand_immediate(imm8: u64, double: bool) -> u64 { + let sign = imm8 >> 7; + let b = (imm8 >> 6) & 1; + let fraction = imm8 & 0x3f; + if double { + (sign << 63) | ((b ^ 1) << 62) | (b * 0xff << 54) | (fraction << 48) + } else { + (sign << 31) | ((b ^ 1) << 30) | (b * 0x1f << 25) | (fraction << 19) + } +} + pub fn decode(word: u32) -> Result { use DecodeError::UnsupportedInstruction; use Instruction::*; - // PAuth and guarded-execution instructions, hint-space forms included, run on the host. - if super::pauth::is_supported(word) || super::gxf::is_supported(word) { - return Ok(Host(word)); - } let width = width(word); let rn = reg(word, 5); let rm = reg(word, 16); @@ -1546,12 +1579,39 @@ pub fn decode(word: u32) -> Result { immediate: ((word >> 10) & 0xfff) << if word & 0x0040_0000 != 0 { 12 } else { 0 }, })); } - if word & 0xffc0_0000 == 0xf980_0000 - || word & 0xffe0_0c00 == 0xf8a0_0800 - || word & 0xffe0_0c00 == 0xf880_0000 - || word & 0xff00_0000 == 0xd800_0000 - { - return Ok(Nop); + // PRFM (unsigned offset), PRFUM (unscaled offset) and PRFM (register). They + // keep their operand and execute as `nop`. + if word & 0xffc0_0000 == 0xf980_0000 { + return Ok(Prefetch(self::Prefetch { + op: rd, + rn, + addressing: Addressing::Offset((((word >> 10) & 0xfff) << 3) as i64), + })); + } + if word & 0xffe0_0c00 == 0xf880_0000 { + return Ok(Prefetch(self::Prefetch { + op: rd, + rn, + addressing: Addressing::Offset(sign_extend(word >> 12, 9)), + })); + } + if word & 0xffe0_0c00 == 0xf8a0_0800 { + return Ok(Prefetch(self::Prefetch { + op: rd, + rn, + addressing: Addressing::Register { + rm, + extend: extension(word >> 13), + amount: if word & 0x1000 != 0 { 3 } else { 0 }, + }, + })); + } + // PRFM (literal). + if word & 0xff00_0000 == 0xd800_0000 { + return Ok(PrefetchLiteral(self::PrefetchLiteral { + op: rd, + offset: sign_extend(word >> 5, 19) << 2, + })); } if word & 0x3f00_0000 == 0x1800_0000 { let size = match word >> 30 { @@ -1858,21 +1918,24 @@ pub fn decode(word: u32) -> Result { })); } if word & 0xffff_f01f == 0xd503_201f { - return Ok(match (word >> 5) & 127 { + let hint = ((word >> 5) & 127) as u8; + return Ok(match hint { + 0 => Nop, 1 => Yield, 2 => Wfe, 3 => Wfi, 4 => Sev, 5 => Sevl, - _ => Nop, + _ => Hint(hint), }); } if word & 0xffff_f01f == 0xd503_301f { + let option = ((word >> 8) & 15) as u8; return match (word >> 5) & 7 { - 2 => Ok(Clrex), - 4 => Ok(Dsb), - 5 => Ok(Dmb), - 6 => Ok(Isb), + 2 => Ok(Clrex(option)), + 4 => Ok(Dsb(option)), + 5 => Ok(Dmb(option)), + 6 => Ok(Isb(option)), _ => Err(UnsupportedInstruction), }; } @@ -1891,11 +1954,18 @@ pub fn decode(word: u32) -> Result { if word & 0xfff8_0000 == 0xd508_0000 { let crn = (word >> 12) & 15; let crm = (word >> 8) & 15; + let sys = SysOp { + op1: ((word >> 16) & 7) as u8, + crn: crn as u8, + crm: crm as u8, + op2: ((word >> 5) & 7) as u8, + rt: rd, + }; if crn == 8 { - return Ok(Tlbi); + return Ok(Tlbi(sys)); } if crn == 7 && matches!(crm, 1 | 5 | 6 | 10 | 11 | 14) { - return Ok(CacheOp); + return Ok(CacheOp(sys)); } return Err(UnsupportedInstruction); } @@ -2086,8 +2156,8 @@ pub fn decode(word: u32) -> Result { acc | (if (imm8 >> bit) & 1 != 0 { 0xff } else { 0 }) << (8 * bit) }), // FMOV: the 8-bit float immediate widened to f32 (twice) or f64. - _ if !op => twice(super::fp::expand_immediate(imm8, false)), - _ if q => super::fp::expand_immediate(imm8, true), + _ if !op => twice(expand_immediate(imm8, false)), + _ if q => expand_immediate(imm8, true), _ => return Err(UnsupportedInstruction), }; // Odd cmode below 12 is ORR/BIC; MOVI becomes MVNI (inverted) with op set. @@ -2121,8 +2191,9 @@ pub fn decode(word: u32) -> Result { } return Ok(SimdCompareZero(self::SimdCompareZero { rd, rn, size, q })); } - // FMOV between a general register and an S/D register: a pure bit move. - if word & 0x7ebe_fc00 == 0x1e26_0000 { + // FMOV between a general register and an S/D register: a pure bit move. Bit 24 + // must be clear; the floating-point 3-source group (FMADD and friends) has it set. + if word & 0x7fbe_fc00 == 0x1e26_0000 { let double64 = word & 0x8000_0000 != 0; let double_type = word & 0x0040_0000 != 0; if double64 != double_type { @@ -2135,11 +2206,6 @@ pub fn decode(word: u32) -> Result { double: double64, })); } - // Scalar floating point and integer SIMD the JIT does not compile: `host::run` - // executes the word. - if super::host::is_supported(word) { - return Ok(Host(word)); - } // Across-lanes reductions: bits 21:17 = 11000, bits 11:10 = 10. They share // the three-same class bits, so they are matched first. ADDV (U=0, opcode // 11011) and, with U=1, UMAXV (01010), UMINV (11010) and UADDLV (00011). @@ -2425,8 +2491,11 @@ pub fn decode(word: u32) -> Result { rd, })); } + // SVC #imm16: the immediate is the whole field. + if word & 0xffe0_001f == 0xd400_0001 { + return Ok(Svc((word >> 5) as u16)); + } match word { - 0xd400_0001 => Ok(Svc), 0xd400_0002 => Ok(Psci), 0xd503_207f => Ok(Wfi), _ => Err(UnsupportedInstruction), @@ -2655,21 +2724,23 @@ mod tests { fn assembler_hints_barriers_and_traps() { use Instruction::*; for (word, expected) in [ - (0xd5033f9f, Dsb), - (0xd5033fbf, Dmb), - (0xd5033fdf, Isb), + (0xd5033f9f, Dsb(0xf)), + (0xd5033fbf, Dmb(0xf)), + (0xd5033fdf, Isb(0xf)), (0xd503201f, Nop), + (0xd503305f, Clrex(0)), (0xd503203f, Yield), (0xd503205f, Wfe), (0xd503207f, Wfi), (0xd503209f, Sev), (0xd50320bf, Sevl), - (0xd503241f, Nop), - (0xd503245f, Nop), - (0xd503249f, Nop), - (0xd50324df, Nop), + (0xd503241f, Hint(32)), + (0xd503245f, Hint(34)), + (0xd503249f, Hint(36)), + (0xd50324df, Hint(38)), (0xd69f03e0, Eret), - (0xd4000001, Svc), + (0xd4000001, Svc(0)), + (0xd4001001, Svc(0x80)), (0xd4000002, Psci), (0xd4200020, Brk(1)), (0xd4210000, Brk(0x800)), @@ -2677,12 +2748,68 @@ mod tests { assert_eq!(decode(word), Ok(expected)); } assert_eq!(decode(0xd50b7423), Ok(DcZva(3))); - assert_eq!(decode(0xd503309f), Ok(Dsb)); - assert!(!Dsb.terminates()); - assert!(Isb.terminates()); + assert_eq!(decode(0xd503309f), Ok(Dsb(0))); + assert!(!Dsb(0).terminates()); + assert!(Isb(0).terminates()); assert!(Eret.terminates()); } #[test] + fn prefetch_and_system_operations_keep_their_operands() { + use Instruction::*; + assert_eq!( + decode(0xf9800420), + Ok(Prefetch(super::Prefetch { + op: 0, + rn: 1, + addressing: Addressing::Offset(8), + })) + ); + assert_eq!( + decode(0xf89f8040), + Ok(Prefetch(super::Prefetch { + op: 0, + rn: 2, + addressing: Addressing::Offset(-8), + })) + ); + assert_eq!( + decode(0xf8a47860), + Ok(Prefetch(super::Prefetch { + op: 0, + rn: 3, + addressing: Addressing::Register { + rm: 4, + extend: Extend::Uxtx, + amount: 3, + }, + })) + ); + assert_eq!( + decode(0xd8000040), + Ok(PrefetchLiteral(super::PrefetchLiteral { op: 0, offset: 8 })) + ); + assert_eq!( + decode(0xd50b7e20), + Ok(CacheOp(super::SysOp { + op1: 3, + crn: 7, + crm: 14, + op2: 1, + rt: 0, + })) + ); + assert_eq!( + decode(0xd508871f), + Ok(Tlbi(super::SysOp { + op1: 0, + crn: 8, + crm: 7, + op2: 0, + rt: 31, + })) + ); + } + #[test] fn logical_immediates_wrap_and_replicate_without_crossing_elements() { for (word, immediate) in [ (0x121c3c21, 0xffff0), @@ -2821,7 +2948,7 @@ mod tests { // modified immediate but have a non-zero `immh`; they are not `movi`. for word in [0x2f08a401, 0x4f255420, 0x4f090420, 0x0f0c8420] { assert!( - matches!(decode(word), Ok(Instruction::Host(_))), + !matches!(decode(word), Ok(Instruction::SimdImm(_))), "{word:08x}" ); } diff --git a/src/target/src/aarch64/disasm.rs b/src/target/src/aarch64/disasm.rs index 0d2247e..a484f6e 100644 --- a/src/target/src/aarch64/disasm.rs +++ b/src/target/src/aarch64/disasm.rs @@ -809,6 +809,9 @@ fn decode(out: &mut String, w: u32, ctx: Option>) { return; } + if ordered_mem(out, w) { + return; + } if mem_imm(out, w) { return; } @@ -876,6 +879,24 @@ fn ror_in_esize(v: u64, r: u32, esize: u32) -> u64 { ((v >> rr) | (v << (esize - rr))) & mask } +/// Load-acquire / store-release register (`ldar{,b,h}`, `stlr{,b,h}`): `[xn]` addressing only. +fn ordered_mem(out: &mut String, w: u32) -> bool { + let mnem = match w & 0x3FFF_FC00 { + 0x08DF_FC00 => "ldar", + 0x089F_FC00 => "stlr", + _ => return false, + }; + let size = w >> 30; + out.push_str(mnem); + out.push_str(["b", "h", "", ""][size as usize]); + out.push(' '); + gp(out, size == 3, rd(w)); + out.push_str(", ["); + sp(out, true, rn(w)); + out.push(']'); + true +} + /// Scaled-unsigned-offset loads/stores (integer and FP). Returns whether it /// matched. fn mem_imm(out: &mut String, w: u32) -> bool { @@ -1499,6 +1520,15 @@ mod tests { // nop is outside the emitted-subset decode coverage: it must surface // as a raw word, never a mis-decode. assert_eq!(one(0xD503_201F), ".word 0xd503201f"); + // ldar/stlr in every width (GNU as output). + assert_eq!(one(0x88df_fc20), "ldar w0, [x1]"); + assert_eq!(one(0xc8df_fc62), "ldar x2, [x3]"); + assert_eq!(one(0x08df_fca4), "ldarb w4, [x5]"); + assert_eq!(one(0x48df_fce6), "ldarh w6, [x7]"); + assert_eq!(one(0x889f_fc20), "stlr w0, [x1]"); + assert_eq!(one(0xc89f_fc62), "stlr x2, [x3]"); + assert_eq!(one(0x089f_fca4), "stlrb w4, [x5]"); + assert_eq!(one(0x489f_fce6), "stlrh w6, [x7]"); } #[test] diff --git a/src/target/src/aarch64/encode.rs b/src/target/src/aarch64/encode.rs index 60dc798..ad35daa 100644 --- a/src/target/src/aarch64/encode.rs +++ b/src/target/src/aarch64/encode.rs @@ -451,6 +451,24 @@ pub fn ldrsh_off(rt: Reg, rn: Reg, off: u16) -> u32 { 0x79C00000 | (((off as u32) >> 1) << 10) | (n(rn) << 5) | n(rt) } +/// Load-acquire register: `size` is the access width in bytes (1, 2, 4 or 8); the value is +/// zero-extended into `rt`. Addressing is a bare base register (`[rn]`) only. +pub fn ldar(rt: Reg, rn: Reg, size: u8) -> u32 { + ordered_word(0x08DF_FC00, size) | (n(rn) << 5) | n(rt) +} + +/// Store-release register: `size` is the access width in bytes (1, 2, 4 or 8). +/// Addressing is a bare base register (`[rn]`) only. +pub fn stlr(rt: Reg, rn: Reg, size: u8) -> u32 { + ordered_word(0x089F_FC00, size) | (n(rn) << 5) | n(rt) +} + +/// `base` (the byte-sized form of an `LDAR`/`STLR`) with the size field for `size` bytes. +fn ordered_word(base: u32, size: u8) -> u32 { + debug_assert!(matches!(size, 1 | 2 | 4 | 8), "ldar/stlr size {size}"); + base | (size.trailing_zeros() << 30) +} + pub fn sdiv(rd: Reg, rn: Reg, rm: Reg) -> u32 { 0x1AC00C00 | (n(rm) << 16) | (n(rn) << 5) | n(rd) } @@ -939,7 +957,11 @@ mod tests { } else { (1u64 << size) - 1 }; - let rotated = ((run >> r) | (run << (size - r))) & mask; + let rotated = if r == 0 { + run + } else { + ((run >> r) | (run << (size - r))) & mask + }; let mut value = 0u64; let mut at = 0; while at < if is64 { 64 } else { 32 } { @@ -1234,6 +1256,19 @@ mod tests { assert_eq!(0x297F10A3, ldp_off_w(Reg::X3, Reg::X4, Reg::X5, -8)); } + #[test] + fn load_acquire_store_release_golden() { + // Checked against GNU as. + assert_eq!(0x88dffc20, ldar(Reg::X0, Reg::X1, 4)); + assert_eq!(0xc8dffc62, ldar(Reg::X2, Reg::X3, 8)); + assert_eq!(0x08dffca4, ldar(Reg::X4, Reg::X5, 1)); + assert_eq!(0x48dffce6, ldar(Reg::X6, Reg::X7, 2)); + assert_eq!(0x889ffc20, stlr(Reg::X0, Reg::X1, 4)); + assert_eq!(0xc89ffc62, stlr(Reg::X2, Reg::X3, 8)); + assert_eq!(0x089ffca4, stlr(Reg::X4, Reg::X5, 1)); + assert_eq!(0x489ffce6, stlr(Reg::X6, Reg::X7, 2)); + } + #[test] fn byte_halfword_offset_golden() { assert_eq!(0x39000C20, strb_off(Reg::X0, Reg::X1, 3)); diff --git a/src/target/src/aarch64/isel_tests.rs b/src/target/src/aarch64/isel_tests.rs index 7fce7fc..7281c34 100644 --- a/src/target/src/aarch64/isel_tests.rs +++ b/src/target/src/aarch64/isel_tests.rs @@ -1,36 +1,65 @@ // Rust selector boundary tests for the Vulcan-derived AArch64 port (Apache-2.0). use super::*; +use volt_ir::attribute::Endianness; use volt_ir::function::{ - Arith, BinOp, CmpOp, Compare, EdgeDesc, Function, Load, Opcode, Ret, Terminator, + Arith, AtomicOrdering, BinOp, Block, CmpOp, Compare, Convert, ConvertKind, Dot, EdgeDesc, + Extract, Function, Load, MemFlags, Opcode, Ret, Terminator, Value, }; -use volt_ir::types::{FloatKind, IntDesc, Type, TypeKind}; +use volt_ir::types::{FloatKind, IntDesc, Type, TypeKind, VectorDesc}; -fn integer(f: &mut Function, signed: bool, bits: u16) -> Type { - f.types.intern(TypeKind::Int(IntDesc { signed, bits })) +fn integer(f: &mut Function, bits: u16) -> Type { + f.types.intern(TypeKind::Int(IntDesc { bits })) +} + +fn convert(f: &mut Function, b: Block, ty: Type, value: Value, kind: ConvertKind) -> Value { + f.append_inst(b, ty, Opcode::Convert(Convert { value, kind })) +} + +/// `x` (an integer of `from` bits) widened to the 64-bit type `wide`, by sign when `signed`; +/// a 64-bit value is already that wide. +fn widen(f: &mut Function, b: Block, wide: Type, x: Value, from: u16, signed: bool) -> Value { + if from == 64 { + return x; + } + let kind = if signed { + ConvertKind::Sext + } else { + ConvertKind::Zext + }; + convert(f, b, wide, x, kind) +} + +/// The low `bits` of the 64-bit `x`. +fn narrow(f: &mut Function, b: Block, ty: Type, x: Value, bits: u16) -> Value { + if bits == 64 { + return x; + } + convert(f, b, ty, x, ConvertKind::Trunc) } #[test] fn emits_native_divide_remainder_and_high_product() { - for signed in [false, true] { - for op in [BinOp::Div, BinOp::Rem, BinOp::Mulh] { - let mut f = Function::new(); - let ty = integer(&mut f, signed, 64); - let b = f.append_block(); - let a = f.append_block_param(b, ty); - let c = f.append_block_param(b, ty); - let result = f.append_inst(b, ty, Opcode::Arith(Arith { op, lhs: a, rhs: c })); - f.set_terminator(b, Terminator::Ret(Ret::one(result))); - let code = compile(&f).unwrap(); - assert_eq!(code.last(), Some(&crate::aarch64::encode::ret())); - let opcode = if op == BinOp::Mulh { - if signed { 0x9b407c00 } else { 0x9bc07c00 } - } else if signed { - 0x9ac00c00 - } else { - 0x9ac00800 - }; - assert!(code.iter().any(|w| w & 0xffe0fc00 == opcode)); - } + for (op, opcode) in [ + (BinOp::UDiv, 0x9ac00800), + (BinOp::SDiv, 0x9ac00c00), + (BinOp::URem, 0x9ac00800), + (BinOp::SRem, 0x9ac00c00), + (BinOp::UMulh, 0x9bc07c00), + (BinOp::SMulh, 0x9b407c00), + ] { + let mut f = Function::new(); + let ty = integer(&mut f, 64); + let b = f.append_block(); + let a = f.append_block_param(b, ty); + let c = f.append_block_param(b, ty); + let result = f.append_inst(b, ty, Opcode::Arith(Arith { op, lhs: a, rhs: c })); + f.set_terminator(b, Terminator::Ret(Ret::one(result))); + let code = compile(&f).unwrap(); + assert_eq!(code.last(), Some(&crate::aarch64::encode::ret()), "{op:?}"); + assert!(code.iter().any(|w| w & 0xffe0fc00 == opcode), "{op:?}"); + // A remainder is the divide followed by a multiply-subtract. + let msub = code.iter().any(|w| w & 0xffe08000 == 0x9b008000); + assert_eq!(msub, matches!(op, BinOp::URem | BinOp::SRem), "{op:?}"); } } @@ -55,7 +84,7 @@ fn scalar_half_and_quad_data_survive_selection() { #[test] fn object_calls_keep_owned_relocations() { let mut f = Function::new(); - let ty = integer(&mut f, false, 64); + let ty = integer(&mut f, 64); let b = f.append_block(); let value = f.append_call(b, ty, "external_target", &[]); f.set_terminator(b, Terminator::Ret(Ret::one(value))); @@ -75,24 +104,28 @@ fn mapped(f: &Function) -> crate::native::JittedFunction { #[cfg(target_arch = "aarch64")] #[test] fn division_edges_execute_with_aarch64_semantics() { - for signed in [false, true] { - for op in [BinOp::Div, BinOp::Rem] { - let mut f = Function::new(); - let ty = integer(&mut f, signed, 64); - let b = f.append_block(); - let a = f.append_block_param(b, ty); - let c = f.append_block_param(b, ty); - let result = f.append_inst(b, ty, Opcode::Arith(Arith { op, lhs: a, rhs: c })); - f.set_terminator(b, Terminator::Ret(Ret::one(result))); - let image = mapped(&f); - // SAFETY: function is generated for precisely this AAPCS64 signature. - let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { image.entry() }; - let zero = unsafe { entry(123, 0) }; - assert_eq!(zero, if op == BinOp::Div { 0 } else { 123 }); - if signed { - let overflow = unsafe { entry(i64::MIN as u64, u64::MAX) }; - assert_eq!(overflow, if op == BinOp::Div { i64::MIN as u64 } else { 0 }); - } + for (op, signed) in [ + (BinOp::UDiv, false), + (BinOp::SDiv, true), + (BinOp::URem, false), + (BinOp::SRem, true), + ] { + let mut f = Function::new(); + let ty = integer(&mut f, 64); + let b = f.append_block(); + let a = f.append_block_param(b, ty); + let c = f.append_block_param(b, ty); + let result = f.append_inst(b, ty, Opcode::Arith(Arith { op, lhs: a, rhs: c })); + f.set_terminator(b, Terminator::Ret(Ret::one(result))); + let image = mapped(&f); + // SAFETY: function is generated for precisely this AAPCS64 signature. + let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { image.entry() }; + let is_div = matches!(op, BinOp::UDiv | BinOp::SDiv); + let zero = unsafe { entry(123, 0) }; + assert_eq!(zero, if is_div { 0 } else { 123 }); + if signed { + let overflow = unsafe { entry(i64::MIN as u64, u64::MAX) }; + assert_eq!(overflow, if is_div { i64::MIN as u64 } else { 0 }); } } } @@ -101,7 +134,7 @@ fn division_edges_execute_with_aarch64_semantics() { #[test] fn loop_edges_execute_parallel_parameter_permutations() { let mut f = Function::new(); - let ty = integer(&mut f, false, 64); + let ty = integer(&mut f, 64); let bool_ty = f.types.intern(TypeKind::Bool); let entry = f.append_block(); let head = f.append_block(); @@ -148,8 +181,8 @@ fn loop_edges_execute_parallel_parameter_permutations() { fn signed_byte_load_widens_without_overreading() { let mut f = Function::new(); let ptr = f.types.ptr_global(); - let byte = integer(&mut f, true, 8); - let wide = integer(&mut f, true, 64); + let byte = integer(&mut f, 8); + let wide = integer(&mut f, 64); let b = f.append_block(); let p = f.append_block_param(b, ptr); let value = f.append_inst( @@ -157,14 +190,13 @@ fn signed_byte_load_widens_without_overreading() { byte, Opcode::Load(Load { ptr: p, - volatile: true, + mem: MemFlags { + volatile: true, + ..MemFlags::new() + }, }), ); - let value = f.append_inst( - b, - wide, - Opcode::Convert(volt_ir::function::Convert { value }), - ); + let value = convert(&mut f, b, wide, value, ConvertKind::Sext); f.set_terminator(b, Terminator::Ret(Ret::one(value))); let image = mapped(&f); let entry: unsafe extern "C" fn(*const i8) -> i64 = unsafe { image.entry() }; @@ -172,6 +204,465 @@ fn signed_byte_load_widens_without_overreading() { assert_eq!(unsafe { entry(&byte) }, -97); } +#[cfg(target_arch = "aarch64")] +#[test] +fn big_endian_loads_are_byte_swapped() { + let bytes: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; + for (bits, expected) in [ + (16u16, u16::from_be_bytes([bytes[0], bytes[1]]) as u64), + ( + 32, + u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64, + ), + (64, u64::from_be_bytes(bytes)), + ] { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = integer(&mut f, bits); + let wide = integer(&mut f, 64); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let mem = MemFlags { + endian: Endianness::Big, + ..MemFlags::new() + }; + let value = f.append_load(b, ty, p, mem); + let value = widen(&mut f, b, wide, value, bits, false); + f.set_terminator(b, Terminator::Ret(Ret::one(value))); + let image = mapped(&f); + let entry: unsafe extern "C" fn(*const u8) -> u64 = unsafe { image.entry() }; + assert_eq!(unsafe { entry(bytes.as_ptr()) }, expected, "{bits}-bit"); + } +} + +#[cfg(target_arch = "aarch64")] +#[test] +fn big_endian_stores_are_byte_swapped() { + for bits in [16u16, 32, 64] { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let x = f.append_block_param(b, wide); + let narrow = narrow(&mut f, b, ty, x, bits); + let mem = MemFlags { + endian: Endianness::Big, + ..MemFlags::new() + }; + f.append_store_mem(b, narrow, p, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let image = mapped(&f); + let entry: unsafe extern "C" fn(*mut u8, u64) = unsafe { image.entry() }; + let mut out = [0u8; 8]; + let x = 0x0123_4567_89ab_cdefu64; + unsafe { entry(out.as_mut_ptr(), x) }; + let n = bits as usize / 8; + assert_eq!(out[..n], x.to_be_bytes()[8 - n..], "{bits}-bit"); + assert!(out[n..].iter().all(|b| *b == 0), "{bits}-bit overwrote"); + } +} + +/// `(p: ptr) -> 64-bit` loading a `bits`-wide integer `offset` bytes past `p` with `mem` and +/// widening it (sign- or zero-extending to match `signed`). +fn ordered_load_fn(bits: u16, signed: bool, mem: MemFlags, offset: i64) -> Function { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = integer(&mut f, bits); + let wide = integer(&mut f, 64); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let addr = if offset == 0 { + p + } else { + f.append_arith_imm(b, ptr, BinOp::Add, p, offset) + }; + let value = f.append_load(b, ty, addr, mem); + let value = widen(&mut f, b, wide, value, bits, signed); + f.set_terminator(b, Terminator::Ret(Ret::one(value))); + f +} + +/// `(p: ptr, x: u64)` storing the low `bits` of `x` `offset` bytes past `p` with `mem`. +fn ordered_store_fn(bits: u16, mem: MemFlags, offset: i64) -> Function { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let x = f.append_block_param(b, wide); + let narrow = narrow(&mut f, b, ty, x, bits); + let addr = if offset == 0 { + p + } else { + f.append_arith_imm(b, ptr, BinOp::Add, p, offset) + }; + f.append_store_mem(b, narrow, addr, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + f +} + +fn with_ordering(ordering: Option, bits: u16) -> MemFlags { + MemFlags { + ordering, + align: u32::from(bits / 8), + ..MemFlags::new() + } +} + +const LOAD_ORDERINGS: [Option; 4] = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Acquire), + Some(AtomicOrdering::SeqCst), +]; +const STORE_ORDERINGS: [Option; 4] = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Release), + Some(AtomicOrdering::SeqCst), +]; + +/// Disassembly of every word `compile` emits for `f`. +fn listing(f: &Function) -> Vec { + compile(f) + .unwrap() + .iter() + .map(|&w| crate::aarch64::disasm::one(w)) + .collect() +} + +fn mnemonic(line: &str) -> &str { + line.split(' ').next().unwrap() +} + +/// Acquire and sequentially consistent loads select `ldar{b,h,}`, release and sequentially +/// consistent stores `stlr{b,h,}`, each addressed through a bare `[xn]` even when an offset +/// could have been folded; relaxed and plain accesses stay ordinary `ldr`/`str`. +#[test] +fn ordered_accesses_select_load_acquire_and_store_release() { + for (bits, load_acq, store_rel, load_plain, store_plain) in [ + (8u16, "ldarb", "stlrb", "ldrb", "strb"), + (16, "ldarh", "stlrh", "ldrh", "strh"), + (32, "ldar", "stlr", "ldr", "str"), + (64, "ldar", "stlr", "ldr", "str"), + ] { + for offset in [0i64, 8] { + for ordering in LOAD_ORDERINGS { + let lines = listing(&ordered_load_fn( + bits, + false, + with_ordering(ordering, bits), + offset, + )); + let ordered = matches!( + ordering, + Some(AtomicOrdering::Acquire | AtomicOrdering::SeqCst) + ); + let want = if ordered { load_acq } else { load_plain }; + let access: Vec<&String> = lines + .iter() + .filter(|l| mnemonic(l).starts_with("ldr") || mnemonic(l).starts_with("ldar")) + .collect(); + assert_eq!(access.len(), 1, "{bits} {ordering:?} {lines:?}"); + assert_eq!(mnemonic(access[0]), want, "{bits} {ordering:?} {lines:?}"); + if ordered { + assert!( + access[0].ends_with(']') && !access[0].contains(", #"), + "{lines:?}" + ); + assert!(!lines.iter().any(|l| mnemonic(l) == load_plain)); + } + } + for ordering in STORE_ORDERINGS { + let lines = listing(&ordered_store_fn( + bits, + with_ordering(ordering, bits), + offset, + )); + let ordered = matches!( + ordering, + Some(AtomicOrdering::Release | AtomicOrdering::SeqCst) + ); + let want = if ordered { store_rel } else { store_plain }; + let access: Vec<&String> = lines + .iter() + .filter(|l| mnemonic(l).starts_with("str") || mnemonic(l).starts_with("stlr")) + .collect(); + assert_eq!(access.len(), 1, "{bits} {ordering:?} {lines:?}"); + assert_eq!(mnemonic(access[0]), want, "{bits} {ordering:?} {lines:?}"); + if ordered { + assert!( + access[0].ends_with(']') && !access[0].contains(", #"), + "{lines:?}" + ); + } + } + } + } +} + +/// An ordering that has no instruction for the access kind, or a type with no ordered form, +/// must fail closed rather than emit a weaker access. +#[test] +fn unorderable_accesses_fail_closed() { + for ordering in [AtomicOrdering::Release, AtomicOrdering::AcqRel] { + let f = ordered_load_fn(32, false, with_ordering(Some(ordering), 32), 0); + assert!( + matches!(compile(&f), Err(Error::Unsupported)), + "{ordering:?}" + ); + } + for ordering in [AtomicOrdering::Acquire, AtomicOrdering::AcqRel] { + let f = ordered_store_fn(32, with_ordering(Some(ordering), 32), 0); + assert!( + matches!(compile(&f), Err(Error::Unsupported)), + "{ordering:?}" + ); + } + // A 24-bit integer has no ordered form, even when relaxed. + let f = ordered_load_fn( + 24, + false, + with_ordering(Some(AtomicOrdering::Relaxed), 0), + 0, + ); + assert!(matches!(compile(&f), Err(Error::Unsupported))); +} + +#[repr(align(16))] +struct Image([u8; 16]); + +const IMAGE: Image = Image([ + 0x81, 0x92, 0xa3, 0xb4, 0xc5, 0xd6, 0xe7, 0xf8, 0x09, 0x1a, 0x2b, 0x3c, 0x4d, 0x5e, 0x6f, 0x70, +]); + +/// Every width, signedness and ordering loads the same value an ordinary load of that type +/// would (zero- or sign-extended), at offset 0 and at a folded-looking offset 8. +#[cfg(target_arch = "aarch64")] +#[test] +fn ordered_loads_produce_the_plain_load_value() { + let image = IMAGE; + for bits in [8u16, 16, 32, 64] { + for signed in [false, true] { + for offset in [0usize, 8] { + let n = usize::from(bits / 8); + let mut raw = [0u8; 8]; + raw[..n].copy_from_slice(&image.0[offset..offset + n]); + let zero = u64::from_le_bytes(raw); + let expected = if signed && bits < 64 { + (((zero << (64 - bits)) as i64) >> (64 - bits)) as u64 + } else { + zero + }; + for ordering in LOAD_ORDERINGS { + let f = + ordered_load_fn(bits, signed, with_ordering(ordering, bits), offset as i64); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*const u8) -> u64 = unsafe { jitted.entry() }; + assert_eq!( + unsafe { entry(image.0.as_ptr()) }, + expected, + "{bits}-bit signed={signed} offset={offset} {ordering:?}" + ); + } + } + } + } +} + +/// Every width and ordering stores exactly its own bytes. +#[cfg(target_arch = "aarch64")] +#[test] +fn ordered_stores_write_exactly_their_width() { + let x = 0x0123_4567_89ab_cdefu64; + for bits in [8u16, 16, 32, 64] { + for offset in [0usize, 8] { + let n = usize::from(bits / 8); + for ordering in STORE_ORDERINGS { + let f = ordered_store_fn(bits, with_ordering(ordering, bits), offset as i64); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*mut u8, u64) = unsafe { jitted.entry() }; + let mut out = Image([0xee; 16]); + unsafe { entry(out.0.as_mut_ptr(), x) }; + let mut want = [0xeeu8; 16]; + want[offset..offset + n].copy_from_slice(&x.to_le_bytes()[..n]); + assert_eq!(out.0, want, "{bits}-bit offset={offset} {ordering:?}"); + } + } + } +} + +/// Float accesses go through an integer register (legalize reinterprets them) and pointers +/// are 64-bit integers. +#[cfg(target_arch = "aarch64")] +#[test] +fn ordered_float_and_pointer_accesses_execute() { + let mem = |ordering, bytes: u32| MemFlags { + ordering: Some(ordering), + align: bytes, + ..MemFlags::new() + }; + for ordering in [AtomicOrdering::Acquire, AtomicOrdering::SeqCst] { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = f.types.intern(TypeKind::Float(FloatKind::F32)); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let v = f.append_load(b, ty, p, mem(ordering, 4)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + assert!(listing(&f).iter().any(|l| mnemonic(l) == "ldar")); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*const f32) -> f32 = unsafe { jitted.entry() }; + let x = -1.5f32; + assert_eq!(unsafe { entry(&x) }, -1.5); + + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = f.types.intern(TypeKind::Float(FloatKind::F64)); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let v = f.append_load(b, ty, p, mem(ordering, 8)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*const f64) -> f64 = unsafe { jitted.entry() }; + let x = 6.25f64; + assert_eq!(unsafe { entry(&x) }, 6.25); + } + for ordering in [AtomicOrdering::Release, AtomicOrdering::SeqCst] { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = f.types.intern(TypeKind::Float(FloatKind::F64)); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let x = f.append_block_param(b, ty); + f.append_store_mem(b, x, p, mem(ordering, 8)); + f.set_terminator(b, Terminator::Ret(Ret::none())); + assert!(listing(&f).iter().any(|l| mnemonic(l) == "stlr")); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*mut f64, f64) = unsafe { jitted.entry() }; + let mut out = 0.0f64; + unsafe { entry(&mut out, -2.75) }; + assert_eq!(out, -2.75); + } + // Pointer-typed: load a pointer acquire, store it back release elsewhere. + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let b = f.append_block(); + let src = f.append_block_param(b, ptr); + let dst = f.append_block_param(b, ptr); + let v = f.append_load(b, ptr, src, mem(AtomicOrdering::Acquire, 8)); + f.append_store_mem(b, v, dst, mem(AtomicOrdering::Release, 8)); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let lines = listing(&f); + assert!(lines.iter().any(|l| mnemonic(l) == "ldar")); + assert!(lines.iter().any(|l| mnemonic(l) == "stlr")); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*const *const u8, *mut *const u8) = unsafe { jitted.entry() }; + let target = 0x1234usize as *const u8; + let mut out = core::ptr::null(); + unsafe { entry(&target, &mut out) }; + assert_eq!(out, target); +} + +/// Message passing: the writer stores data, then a release flag; the reader acquire-loads the +/// flag and then reads the data in the same compiled function. A reader that observes round +/// `i` must observe round `i`'s data. An ack handshake keeps the writer from running ahead, so +/// the expected data is exact on every round. +#[cfg(target_arch = "aarch64")] +#[test] +fn release_acquire_message_passing_between_threads() { + use core::sync::atomic::{AtomicU64, Ordering}; + + // writer(data, flag, value, round): *data = value; release-store *flag = round. + let writer = { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let u64t = integer(&mut f, 64); + let b = f.append_block(); + let data = f.append_block_param(b, ptr); + let flag = f.append_block_param(b, ptr); + let value = f.append_block_param(b, u64t); + let round = f.append_block_param(b, u64t); + f.append_store_mem(b, value, data, with_ordering(None, 64)); + f.append_store_mem( + b, + round, + flag, + with_ordering(Some(AtomicOrdering::Release), 64), + ); + f.set_terminator(b, Terminator::Ret(Ret::none())); + f + }; + // reader(flag, data, out) -> flag: acquire-load the flag, then load the data into *out. + let reader = { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let u64t = integer(&mut f, 64); + let b = f.append_block(); + let flag = f.append_block_param(b, ptr); + let data = f.append_block_param(b, ptr); + let out = f.append_block_param(b, ptr); + let seen = f.append_load( + b, + u64t, + flag, + with_ordering(Some(AtomicOrdering::Acquire), 64), + ); + let value = f.append_load(b, u64t, data, with_ordering(None, 64)); + f.append_store_mem(b, value, out, with_ordering(None, 64)); + f.set_terminator(b, Terminator::Ret(Ret::one(seen))); + f + }; + let writer_image = mapped(&writer); + let reader_image = mapped(&reader); + let write: unsafe extern "C" fn(*mut u64, *mut u64, u64, u64) = unsafe { writer_image.entry() }; + let read: unsafe extern "C" fn(*const u64, *const u64, *mut u64) -> u64 = + unsafe { reader_image.entry() }; + + const ROUNDS: u64 = 1000; + let expected = |round: u64| round.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + let mut data = 0u64; + let mut flag = 0u64; + // The raw addresses cross the thread boundary as integers; each is touched only by the + // compiled code (and the final reads happen after the scope joins). + let (data_addr, flag_addr) = (&raw mut data as usize, &raw mut flag as usize); + let ack = AtomicU64::new(0); + std::thread::scope(|scope| { + scope.spawn(|| { + for round in 1..=ROUNDS { + unsafe { + write( + data_addr as *mut u64, + flag_addr as *mut u64, + expected(round), + round, + ) + }; + while ack.load(Ordering::SeqCst) != round { + std::thread::yield_now(); + } + } + }); + let mut out = 0u64; + for round in 1..=ROUNDS { + loop { + let seen = + unsafe { read(flag_addr as *const u64, data_addr as *const u64, &mut out) }; + if seen == round { + break; + } + assert!(seen < round, "flag ran ahead of the ack handshake"); + std::thread::yield_now(); + } + assert_eq!(out, expected(round), "round {round}"); + ack.store(round, Ordering::SeqCst); + } + }); +} + /// A compiled `u64 -> u64` function and the mapping that keeps it alive. #[cfg(target_arch = "aarch64")] struct NarrowOp { @@ -183,16 +674,18 @@ struct NarrowOp { /// register operand from an `Iconst`, to run both lowerings on the same input. #[cfg(target_arch = "aarch64")] fn narrow_op(signed: bool, bits: u16, op: BinOp, k: i64, immediate: bool) -> NarrowOp { + // The right shift of a signed value is arithmetic. + let op = if signed && op == BinOp::Shr { + BinOp::Sar + } else { + op + }; let mut f = Function::new(); - let wide = integer(&mut f, false, 64); - let ty = integer(&mut f, signed, bits); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); let b = f.append_block(); let x = f.append_block_param(b, wide); - let narrow = f.append_inst( - b, - ty, - Opcode::Convert(volt_ir::function::Convert { value: x }), - ); + let narrow = narrow(&mut f, b, ty, x, bits); let result = if immediate { f.append_arith_imm(b, ty, op, narrow, k) } else { @@ -207,11 +700,7 @@ fn narrow_op(signed: bool, bits: u16, op: BinOp, k: i64, immediate: bool) -> Nar }), ) }; - let back = f.append_inst( - b, - wide, - Opcode::Convert(volt_ir::function::Convert { value: result }), - ); + let back = widen(&mut f, b, wide, result, bits, signed); f.set_terminator(b, Terminator::Ret(Ret::one(back))); let image = mapped(&f); // SAFETY: generated for exactly this AAPCS64 signature; `image` outlives the call. @@ -365,12 +854,614 @@ fn narrow_results_stay_canonical_for_later_uses() { // SAFETY: generated for this signature. unsafe { (f.call)(x) } }; - // u8: a shift that overflows the type wraps to 8 bits. + // i8: a shift that overflows the type wraps to 8 bits. assert_eq!(run(false, 8, BinOp::Shl, 4, 0xff), 0xf0); assert_eq!(run(false, 8, BinOp::Shl, 4, 0x10f), 0xf0); - // u8 & and >> keep the value inside 8 bits. + // i8 & and >> keep the value inside 8 bits. assert_eq!(run(false, 8, BinOp::BitAnd, 0x70, 0xabcd), 0xcd & 0x70); assert_eq!(run(false, 8, BinOp::Shr, 4, 0x1ab), 0xa); - // i8 keeps its sign through an arithmetic shift. + // A signed i8 keeps its sign through an arithmetic shift. assert_eq!(run(true, 8, BinOp::Shr, 2, 0xf0) as u8, 0xfc); } + +/// Signed and unsigned behavior is chosen by the operation: each pair of operations below sees +/// the same bits and must differ exactly where the sign matters. +#[cfg(target_arch = "aarch64")] +mod signedness { + use super::*; + + const WIDTHS: [u16; 4] = [8, 16, 32, 64]; + + fn mask(bits: u16) -> u64 { + if bits == 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + } + } + + /// `v`'s low `bits` as a signed number. + fn sx(bits: u16, v: u64) -> i64 { + let shift = 64 - u32::from(bits); + ((v << shift) as i64) >> shift + } + + /// Values of a `bits`-wide integer (zero-extended) around every sign boundary. + fn samples(bits: u16) -> Vec { + let m = mask(bits); + let max = m >> 1; + let mut v = vec![ + 0, + 1, + 2, + 3, + 7, + 100, + 0x55 & m, + 0xa5 & m, + 127 & m, + 128 & m, + 200 & m, + max, + max + 1, + m - 1, + m, + 0x0123_4567_89ab_cdef & m, + 0xfedc_ba98_7654_3210 & m, + ]; + v.sort_unstable(); + v.dedup(); + v + } + + /// The result of `op` on the `bits`-wide operands `a` and `b` (zero-extended), zero-extended. + fn model(op: BinOp, bits: u16, a: u64, b: u64) -> u64 { + let m = mask(bits); + let (sa, sb) = (i128::from(sx(bits, a)), i128::from(sx(bits, b))); + let r = match op { + BinOp::UDiv => { + if b == 0 { + 0 + } else { + a / b + } + } + BinOp::SDiv => { + if b == 0 { + 0 + } else { + (sa / sb) as u64 + } + } + BinOp::URem => { + if b == 0 { + a + } else { + a % b + } + } + BinOp::SRem => { + if b == 0 { + a + } else { + (sa % sb) as u64 + } + } + BinOp::Shr => a >> b, + BinOp::Sar => (sx(bits, a) >> b) as u64, + BinOp::UMulh => ((u128::from(a) * u128::from(b)) >> bits) as u64, + BinOp::SMulh => ((sa * sb) >> bits) as u64, + _ => unreachable!("{op:?}"), + }; + r & m + } + + /// `(x: u64, y: u64) -> u64`: `op` on the low `bits` of both, zero-extended back. + fn binary_fn(bits: u16, op: BinOp) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let y = f.append_block_param(b, wide); + let nx = narrow(&mut f, b, ty, x, bits); + let ny = narrow(&mut f, b, ty, y, bits); + let r = f.append_inst( + b, + ty, + Opcode::Arith(Arith { + op, + lhs: nx, + rhs: ny, + }), + ); + let r = widen(&mut f, b, wide, r, bits, false); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + f + } + + /// `(x: u64) -> u64`: `x op k` with `k` an `ArithImm` constant. + fn immediate_fn(bits: u16, op: BinOp, k: i64) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let nx = narrow(&mut f, b, ty, x, bits); + let r = f.append_arith_imm(b, ty, op, nx, k); + let r = widen(&mut f, b, wide, r, bits, false); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + f + } + + fn call2(image: &crate::native::JittedFunction, x: u64, y: u64) -> u64 { + // SAFETY: every caller maps a function built for this signature. + let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { image.entry() }; + unsafe { entry(x, y) } + } + + fn call1(image: &crate::native::JittedFunction, x: u64) -> u64 { + // SAFETY: every caller maps a function built for this signature. + let entry: unsafe extern "C" fn(u64) -> u64 = unsafe { image.entry() }; + unsafe { entry(x) } + } + + /// Runs `op` on every pair of samples, with the right operand in a register and (when + /// `immediates`) as a sign-extended `ArithImm`, against `model`. + fn check_binary(ops: &[BinOp], amounts: Option Vec>, immediates: bool) { + for &bits in &WIDTHS { + for &op in ops { + let rights = amounts.map_or_else(|| samples(bits), |f| f(bits)); + let image = mapped(&binary_fn(bits, op)); + for &b in &rights { + let imm = immediates.then(|| mapped(&immediate_fn(bits, op, sx(bits, b)))); + for &a in &samples(bits) { + let want = model(op, bits, a, b); + // The operands carry junk above their width: only the type counts. + let junk = !mask(bits); + let got = call2(&image, a | junk, b | junk); + assert_eq!(got, want, "{op:?} i{bits} {a:#x}, {b:#x}"); + if let Some(imm) = &imm { + let got = call1(imm, a | junk); + assert_eq!(got, want, "{op:?} i{bits} {a:#x}, #{b:#x}"); + } + } + } + } + } + } + + #[test] + fn division_and_remainder_differ_by_operation() { + check_binary( + &[BinOp::UDiv, BinOp::SDiv, BinOp::URem, BinOp::SRem], + None, + true, + ); + // -7 / 2 and -7 % 2 for i8: signed and unsigned disagree, and both are exact. + let (a, b) = (0xf9u64, 2u64); + assert_eq!(model(BinOp::SDiv, 8, a, b), 0xfd); + assert_eq!(model(BinOp::UDiv, 8, a, b), 124); + assert_eq!(model(BinOp::SRem, 8, a, b), 0xff); + assert_eq!(model(BinOp::URem, 8, a, b), 1); + } + + #[test] + fn right_shifts_differ_by_operation() { + fn amounts(bits: u16) -> Vec { + [0, 1, 3, 7, u64::from(bits / 2), u64::from(bits) - 1] + .into_iter() + .filter(|s| *s < u64::from(bits)) + .collect() + } + check_binary(&[BinOp::Shr, BinOp::Sar], Some(amounts), true); + let image = mapped(&binary_fn(8, BinOp::Sar)); + assert_eq!(call2(&image, 0x80, 7), 0xff); + let image = mapped(&binary_fn(8, BinOp::Shr)); + assert_eq!(call2(&image, 0x80, 7), 1); + } + + #[test] + fn high_products_differ_by_operation() { + check_binary(&[BinOp::UMulh, BinOp::SMulh], None, true); + // i64: (-1) * (-1) has high half 0 signed, but 0xffff_ffff_ffff_fffe unsigned. + let image = mapped(&binary_fn(64, BinOp::SMulh)); + assert_eq!(call2(&image, u64::MAX, u64::MAX), 0); + let image = mapped(&binary_fn(64, BinOp::UMulh)); + assert_eq!(call2(&image, u64::MAX, u64::MAX), u64::MAX - 1); + } + + /// `(x, y) -> (x cmp y) as u64` on the low `bits`; branching on the compare when `branch`, + /// so the compare-and-branch fusion sees it. + fn compare_fn(bits: u16, op: CmpOp, branch: bool) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let bool_ty = f.types.intern(TypeKind::Bool); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let y = f.append_block_param(b, wide); + let nx = narrow(&mut f, b, ty, x, bits); + let ny = narrow(&mut f, b, ty, y, bits); + let cond = f.append_inst( + b, + bool_ty, + Opcode::Icmp(Compare { + op, + lhs: nx, + rhs: ny, + }), + ); + if branch { + let yes = f.append_block(); + let no = f.append_block(); + f.append_if(b, cond, EdgeDesc::bare(yes), EdgeDesc::bare(no)); + let one = f.append_inst(yes, wide, Opcode::Iconst(1)); + f.set_terminator(yes, Terminator::Ret(Ret::one(one))); + let zero = f.append_inst(no, wide, Opcode::Iconst(0)); + f.set_terminator(no, Terminator::Ret(Ret::one(zero))); + } else { + let r = convert(&mut f, b, wide, cond, ConvertKind::Zext); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + } + f + } + + fn compare_model(op: CmpOp, bits: u16, a: u64, b: u64) -> bool { + let (sa, sb) = (sx(bits, a), sx(bits, b)); + match op { + CmpOp::Eq => a == b, + CmpOp::Ne => a != b, + CmpOp::Slt => sa < sb, + CmpOp::Sle => sa <= sb, + CmpOp::Sgt => sa > sb, + CmpOp::Sge => sa >= sb, + CmpOp::Ult => a < b, + CmpOp::Ule => a <= b, + CmpOp::Ugt => a > b, + CmpOp::Uge => a >= b, + _ => unreachable!("{op:?}"), + } + } + + #[test] + fn integer_compares_differ_by_operation() { + let ops = [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Slt, + CmpOp::Sle, + CmpOp::Sgt, + CmpOp::Sge, + CmpOp::Ult, + CmpOp::Ule, + CmpOp::Ugt, + CmpOp::Uge, + ]; + for bits in WIDTHS { + for op in ops { + for branch in [false, true] { + let image = mapped(&compare_fn(bits, op, branch)); + let junk = !mask(bits); + for &a in &samples(bits) { + for &b in &samples(bits) { + let got = call2(&image, a | junk, b | junk); + let want = u64::from(compare_model(op, bits, a, b)); + assert_eq!( + got, want, + "{op:?} i{bits} {a:#x}, {b:#x} (branch: {branch})" + ); + } + } + } + } + } + } + + #[test] + fn pointer_compares_are_unsigned() { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let wide = integer(&mut f, 64); + let bool_ty = f.types.intern(TypeKind::Bool); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let q = f.append_block_param(b, ptr); + let cond = f.append_inst( + b, + bool_ty, + Opcode::Icmp(Compare { + op: CmpOp::Ult, + lhs: p, + rhs: q, + }), + ); + let r = convert(&mut f, b, wide, cond, ConvertKind::Zext); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + let image = mapped(&f); + assert_eq!(call2(&image, 1, u64::MAX), 1); + assert_eq!(call2(&image, u64::MAX, 1), 0); + } + + /// `(x: u64) -> u64`: `x` truncated to `from` bits, converted to `to` bits with `kind`, and + /// zero-extended to 64 bits. + fn int_convert_fn(kind: ConvertKind, from: u16, to: u16) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let a = integer(&mut f, from); + let c = integer(&mut f, to); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let x = narrow(&mut f, b, a, x, from); + let y = convert(&mut f, b, c, x, kind); + let y = widen(&mut f, b, wide, y, to, false); + f.set_terminator(b, Terminator::Ret(Ret::one(y))); + f + } + + #[test] + fn extensions_and_truncations_differ_by_kind() { + for from in WIDTHS { + for to in WIDTHS { + if from < to { + for kind in [ConvertKind::Zext, ConvertKind::Sext] { + let image = mapped(&int_convert_fn(kind, from, to)); + for &x in &samples(from) { + let want = if kind == ConvertKind::Sext { + sx(from, x) as u64 & mask(to) + } else { + x + }; + let got = call1(&image, x | !mask(from)); + assert_eq!(got, want, "{kind:?} i{from} -> i{to} {x:#x}"); + } + } + } else if from > to { + let image = mapped(&int_convert_fn(ConvertKind::Trunc, from, to)); + for &x in &samples(from) { + let got = call1(&image, x); + assert_eq!(got, x & mask(to), "Trunc i{from} -> i{to} {x:#x}"); + } + } + } + } + } + + const FLOAT_INPUTS: [f64; 23] = [ + 0.0, + 1.5, + -1.5, + 127.0, + 128.0, + 200.0, + -129.0, + 255.9, + 256.0, + 40000.0, + 70000.0, + -70000.0, + 2147483647.0, + 2147483648.0, + 4294967295.0, + 4294967296.0, + -1.0e10, + 1.0e10, + 9.3e18, + -9.3e18, + 1.0e30, + -1.0e30, + f64::NAN, + ]; + + fn float_type(f: &mut Function, double: bool) -> Type { + f.types.intern(TypeKind::Float(if double { + FloatKind::F64 + } else { + FloatKind::F32 + })) + } + + /// `(x: u64) -> float`: the low `bits` of `x`, converted with `kind`. + fn int_to_float_fn(kind: ConvertKind, bits: u16, double: bool) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let fty = float_type(&mut f, double); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let x = narrow(&mut f, b, ty, x, bits); + let y = convert(&mut f, b, fty, x, kind); + f.set_terminator(b, Terminator::Ret(Ret::one(y))); + f + } + + /// `(x: float) -> u64`: `x` converted to `bits` with `kind`, zero-extended. + fn float_to_int_fn(kind: ConvertKind, bits: u16, double: bool) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let fty = float_type(&mut f, double); + let b = f.append_block(); + let x = f.append_block_param(b, fty); + let y = convert(&mut f, b, ty, x, kind); + let y = widen(&mut f, b, wide, y, bits, false); + f.set_terminator(b, Terminator::Ret(Ret::one(y))); + f + } + + #[test] + fn integer_to_float_differs_by_kind() { + let inputs = { + let mut v = Vec::new(); + for bits in WIDTHS { + v.extend(samples(bits)); + } + v + }; + for bits in WIDTHS { + for kind in [ConvertKind::SiToFp, ConvertKind::UiToFp] { + let signed = kind == ConvertKind::SiToFp; + let single = mapped(&int_to_float_fn(kind, bits, false)); + let double = mapped(&int_to_float_fn(kind, bits, true)); + // SAFETY: both were built for these signatures. + let single: unsafe extern "C" fn(u64) -> f32 = unsafe { single.entry() }; + let double: unsafe extern "C" fn(u64) -> f64 = unsafe { double.entry() }; + for &x in &inputs { + let v = x & mask(bits); + let (want32, want64) = if signed { + (sx(bits, v) as f32, sx(bits, v) as f64) + } else { + (v as f32, v as f64) + }; + // Garbage above the type must not matter. + let arg = x | !mask(bits); + let arg = if bits == 64 { x } else { arg }; + let (got32, got64) = unsafe { (single(arg), double(arg)) }; + assert_eq!(got32, want32, "{kind:?} i{bits} {x:#x} -> f32"); + assert_eq!(got64, want64, "{kind:?} i{bits} {x:#x} -> f64"); + } + } + } + } + + #[test] + fn float_to_integer_differs_by_kind() { + for bits in WIDTHS { + for kind in [ConvertKind::FpToSi, ConvertKind::FpToUi] { + let signed = kind == ConvertKind::FpToSi; + let single = mapped(&float_to_int_fn(kind, bits, false)); + let double = mapped(&float_to_int_fn(kind, bits, true)); + // SAFETY: both were built for these signatures. + let single: unsafe extern "C" fn(f32) -> u64 = unsafe { single.entry() }; + let double: unsafe extern "C" fn(f64) -> u64 = unsafe { double.entry() }; + let want = |narrow32: u64, wide64: u64| { + // A 64-bit destination saturates at 64 bits; every narrower one keeps the + // low bits of the 32-bit saturating conversion. + if bits == 64 { + wide64 + } else { + narrow32 & mask(bits) + } + }; + for &x in &FLOAT_INPUTS { + let (w32, w64) = if signed { + ( + u64::from((x as f32 as i32) as u32), + (x as f32 as i64) as u64, + ) + } else { + (u64::from(x as f32 as u32), x as f32 as u64) + }; + assert_eq!( + unsafe { single(x as f32) }, + want(w32, w64), + "{kind:?} f32 {x} -> i{bits}" + ); + let (w32, w64) = if signed { + (u64::from((x as i32) as u32), (x as i64) as u64) + } else { + (u64::from(x as u32), x as u64) + }; + assert_eq!( + unsafe { double(x) }, + want(w32, w64), + "{kind:?} f64 {x} -> i{bits}" + ); + } + } + } + } + + #[test] + fn float_resize_extends_and_truncates() { + let build = |from_double: bool| { + let mut f = Function::new(); + let a = float_type(&mut f, from_double); + let c = float_type(&mut f, !from_double); + let b = f.append_block(); + let x = f.append_block_param(b, a); + let y = convert(&mut f, b, c, x, ConvertKind::FpResize); + f.set_terminator(b, Terminator::Ret(Ret::one(y))); + mapped(&f) + }; + let widen = build(false); + let shrink = build(true); + // SAFETY: built for these signatures. + let widen: unsafe extern "C" fn(f32) -> f64 = unsafe { widen.entry() }; + let shrink: unsafe extern "C" fn(f64) -> f32 = unsafe { shrink.entry() }; + for x in [0.0f64, -1.5, 1.0e10, 3.4e38, 1.0e300, f64::INFINITY] { + assert_eq!(unsafe { shrink(x) }, x as f32); + assert_eq!(unsafe { widen(x as f32) }, f64::from(x as f32)); + } + } + + /// Integer vector lanes are held zero-extended; widening them is up to the program. + #[test] + fn vector_lanes_extend_by_explicit_conversion() { + for (kind, want) in [ + (ConvertKind::Sext, 0xffff_ffff_8000_0000u64), + (ConvertKind::Zext, 0x8000_0000), + ] { + let mut f = Function::new(); + let i32t = integer(&mut f, 32); + let wide = integer(&mut f, 64); + let vt = f + .types + .intern(TypeKind::Vector(VectorDesc { len: 4, elem: i32t })); + let b = f.append_block(); + let p: Vec = (0..4).map(|_| f.append_block_param(b, i32t)).collect(); + let v = f.append_struct_new(b, vt, &p); + let lane = f.append_inst( + b, + i32t, + Opcode::Extract(Extract { + aggregate: v, + index: 2, + }), + ); + let r = convert(&mut f, b, wide, lane, kind); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + let image = mapped(&f); + // SAFETY: built for this signature. + let entry: unsafe extern "C" fn(u32, u32, u32, u32) -> u64 = unsafe { image.entry() }; + assert_eq!(unsafe { entry(1, 2, 0x8000_0000, 4) }, want, "{kind:?}"); + } + } + + #[test] + fn dot_selects_signed_or_unsigned_by_flag() { + for signed in [false, true] { + let mut f = Function::new(); + let i8t = integer(&mut f, 8); + let i32t = integer(&mut f, 32); + let bytes = f + .types + .intern(TypeKind::Vector(VectorDesc { len: 16, elem: i8t })); + let acc_ty = f + .types + .intern(TypeKind::Vector(VectorDesc { len: 4, elem: i32t })); + let b = f.append_block(); + let acc = f.append_block_param(b, acc_ty); + let x = f.append_block_param(b, bytes); + let y = f.append_block_param(b, bytes); + let r = f.append_inst( + b, + acc_ty, + Opcode::Dot(Dot { + acc, + a: x, + b: y, + signed, + }), + ); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + let code = compile(&f).unwrap(); + let v = crate::aarch64::encode::Reg::from_u5(0); + let sdot = crate::aarch64::encode::sdot(v, v, v); + let udot = crate::aarch64::encode::udot(v, v, v); + let find = |word: u32| code.iter().any(|w| w & 0xffe0_fc00 == word & 0xffe0_fc00); + assert_eq!(find(sdot), signed, "signed={signed}"); + assert_eq!(find(udot), !signed, "signed={signed}"); + } + } +} diff --git a/src/target/src/aarch64/jit.rs b/src/target/src/aarch64/jit.rs index c2359ad..2b61064 100644 --- a/src/target/src/aarch64/jit.rs +++ b/src/target/src/aarch64/jit.rs @@ -41,3 +41,88 @@ impl Compiled { unsafe { self.mapping.entry_at(offset) } } } + +#[cfg(all(test, target_arch = "aarch64"))] +mod tests { + use super::*; + use crate::aarch64::{isel::ModelCaps, link::compile_module}; + use volt_ir::function::{ + ArithImm, BinOp, Function, GlobalAddr, MemFlags, Opcode, Ret, Terminator, + }; + use volt_ir::module::{DataReloc, Module}; + + #[test] + fn linked_module_maps_calls_globals_and_data_relocations() { + // inc(x) = x + 1 + let mut inc = Function::new(); + let u64_t = inc.types.parse_type("i64").unwrap(); + let b = inc.append_block(); + let x = inc.append_block_param(b, u64_t); + let r = inc.append_inst( + b, + u64_t, + Opcode::ArithImm(ArithImm { + op: BinOp::Add, + lhs: x, + imm: 1, + }), + ); + inc.set_terminator(b, Terminator::Ret(Ret::one(r))); + // twice(x) = inc(inc(x)) + let mut twice = Function::new(); + let u64_t = twice.types.parse_type("i64").unwrap(); + let b = twice.append_block(); + let x = twice.append_block_param(b, u64_t); + let once = twice.append_call(b, u64_t, "inc", &[x]); + let r = twice.append_call(b, u64_t, "inc", &[once]); + twice.set_terminator(b, Terminator::Ret(Ret::one(r))); + // seed() = *seed_value + let mut seed = Function::new(); + let u64_t = seed.types.parse_type("i64").unwrap(); + let ptr_t = seed.types.parse_type("ptr").unwrap(); + let b = seed.append_block(); + let symbol = seed.intern_symbol("seed_value"); + let addr = seed.append_inst( + b, + ptr_t, + Opcode::GlobalAddr(GlobalAddr { + symbol, + via_got: false, + }), + ); + let v = seed.append_load(b, u64_t, addr, MemFlags::new()); + seed.set_terminator(b, Terminator::Ret(Ret::one(v))); + + let mut module = Module::new(); + module.add_function("twice", twice); + module.add_function("inc", inc); + module.add_function("seed", seed); + module.add_writable("seed_value", 40u64.to_le_bytes().to_vec()); + module.add_data_relocs( + "ptrs", + vec![0; 8], + vec![DataReloc { + offset: 0, + symbol: "inc".into(), + }], + ); + let linked = compile_module(&module, &ModelCaps::default()).unwrap(); + let compiled = Compiled::new(linked).unwrap(); + // SAFETY: signatures match the IR above and the host is AArch64. + unsafe { + let twice: extern "C" fn(u64) -> u64 = compiled.entry("twice").unwrap(); + assert_eq!(twice(5), 7); + let seed: extern "C" fn() -> u64 = compiled.entry("seed").unwrap(); + assert_eq!(seed(), 40); + let inc: extern "C" fn(u64) -> u64 = compiled.entry("inc").unwrap(); + let slot = compiled + .data_addr("ptrs") + .unwrap() + .cast::() + .read_unaligned(); + assert_eq!(slot, inc as usize as u64); + } + assert_eq!(compiled.offset_of("twice"), Some(0)); + assert!(!compiled.code().is_empty()); + } +} diff --git a/src/target/src/aarch64/link.rs b/src/target/src/aarch64/link.rs index 92354c9..4e13c5d 100644 --- a/src/target/src/aarch64/link.rs +++ b/src/target/src/aarch64/link.rs @@ -1,16 +1,18 @@ -//! AArch64 static/JIT linker: concatenate pre-compiled functions, resolve -//! intra-module calls, lay out data globals, and carry `global_addr` -//! relocations forward. +//! AArch64 static/JIT linker: compile a [`volt_ir::module::Module`], +//! concatenate its functions, resolve intra-module calls, lay out data +//! globals, and carry `global_addr` relocations forward. //! -//! [`Module`] takes pre-compiled [`Vec`] function bodies. Compile a -//! `volt_ir::function::Function` through [`super::isel::compile_function`] -//! first, retaining relocations for object emission or native module mapping. +//! Each function goes through [`super::isel::compile_function`]; the +//! relocations it retains drive linking here and, for object emission or native +//! module mapping, the unresolved ones are carried in [`Linked`]. -use alloc::string::{String, ToString as _}; +use alloc::string::String; use alloc::vec::Vec; +use volt_ir::module::{DataKind, DataReloc, Module}; + use super::encode; -use super::isel::{Reloc, RelocKind}; +use super::isel::{self, ModelCaps, Reloc, RelocKind}; /// Linker failure. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -36,112 +38,6 @@ impl core::fmt::Display for Error { impl core::error::Error for Error {} -/// The section a data global lands in: read-only, writable, or zero-initialized. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum DataKind { - Rodata, - Data, - Bss, -} - -/// An internal relocation within a data object. At byte offset `off` within -/// the object, the linker must write a pointer-sized absolute runtime address -/// of `symbol` once the module is mapped. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct DataReloc { - pub off: usize, - pub symbol: String, -} - -/// A named data global placed in the linked output. For `.bss`, `bytes` is -/// empty and `size` gives the zero-initialized length; otherwise -/// `size == bytes.len` as `u64`. -#[derive(Clone, Debug)] -pub struct Data { - pub name: String, - pub bytes: Vec, - pub kind: DataKind, - pub size: u64, - pub relocs: Vec, -} - -/// A set of named functions and data globals to link together. The first added -/// function is the entry point at offset 0 of the linked image. -#[derive(Clone, Debug, Default)] -pub struct Module { - pub functions: Vec<(String, Vec)>, - pub data: Vec, -} - -impl Module { - pub fn new() -> Module { - Module::default() - } - - pub fn add_function(&mut self, name: &str, code: Vec) { - self.functions.push((name.to_string(), code)); - } - - /// Add a named read-only data blob (a global constant) into `.rodata`. - pub fn add_data(&mut self, name: &str, bytes: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.to_string(), - bytes, - kind: DataKind::Rodata, - size, - relocs: Vec::new(), - }); - } - - /// Add a named read-only data blob carrying internal relocations. - pub fn add_data_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.to_string(), - bytes, - kind: DataKind::Rodata, - size, - relocs, - }); - } - - /// Add a named writable data global into `.data`. - pub fn add_writable(&mut self, name: &str, bytes: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.to_string(), - bytes, - kind: DataKind::Data, - size, - relocs: Vec::new(), - }); - } - - /// Add a named writable data global carrying internal relocations. - pub fn add_writable_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.to_string(), - bytes, - kind: DataKind::Data, - size, - relocs, - }); - } - - /// Add a named zero-initialized data global of `size` bytes into `.bss`. - pub fn add_bss(&mut self, name: &str, size: u64) { - self.data.push(Data { - name: name.to_string(), - bytes: Vec::new(), - kind: DataKind::Bss, - size, - relocs: Vec::new(), - }); - } -} - /// A function's byte offset within the linked image. #[derive(Clone, Debug, Eq, PartialEq)] pub struct Symbol { @@ -157,6 +53,8 @@ pub struct DataSym { pub kind: DataKind, pub off: usize, pub size: usize, + /// The global's requested alignment (`0` = natural), as in [`volt_ir::module::Global::align`]. + pub align: u32, pub bytes: Vec, pub relocs: Vec, } @@ -182,14 +80,6 @@ impl Linked { } } -/// A data object's natural alignment: the next power of two up to `size`, -/// capped at 8. Covers scalars and pointers up to a 64-bit word. -pub fn align_of_data(size: u64) -> usize { - let size = size.max(1); - let pow = size.next_power_of_two(); - pow.min(8) as usize -} - fn align_forward(off: usize, align: usize) -> usize { let mask = align - 1; (off + mask) & !mask @@ -228,36 +118,42 @@ pub fn apply_global_reloc(code: &mut [u32], r: &Reloc, site_addr: usize, target_ } } -/// Concatenate the module's functions, resolve each intra-module `call` -/// relocation to a PC-relative `bl`, lay out data globals into per-section -/// `DataSym`s, and carry every `global_addr` relocation forward unresolved. +/// Compile every function of `module` with `caps`, concatenate the code, +/// resolve each intra-module `call` relocation to a PC-relative `bl`, lay out +/// data globals into per-section `DataSym`s, and carry every `global_addr` +/// relocation forward unresolved. /// -/// `relocs` are supplied alongside the module: `relocs[i]` holds the isel -/// relocations for `module.functions[i]`. -pub fn compile_module(module: &Module, relocs: &[Vec]) -> Result { - let n = module.functions.len(); +/// The first function sits at offset 0 of the image. Symbols the module does +/// not define are an error for calls ([`Error::UndefinedSymbol`]) but are left +/// unresolved for `global_addr` relocations and data relocations. +pub fn compile_module(module: &Module, caps: &ModelCaps) -> Result { + let compiled = module + .functions + .iter() + .map(|def| isel::compile_function(&def.function, caps).map_err(|_| Error::Unsupported)) + .collect::, _>>()?; + let n = compiled.len(); let mut word_off = Vec::with_capacity(n); let mut total = 0usize; - for (_, code) in module.functions.iter() { + for part in &compiled { word_off.push(total); - total += code.len(); + total += part.code.len(); } let mut code: Vec = Vec::with_capacity(total); - for (_, words) in module.functions.iter() { - code.extend_from_slice(words); + for part in &compiled { + code.extend_from_slice(&part.code); } let mut global_relocs: Vec = Vec::new(); - for (i, rs) in relocs.iter().enumerate() { - for r in rs { + for (i, part) in compiled.iter().enumerate() { + for r in &part.relocs { let at = word_off[i] + r.offset; match r.kind { RelocKind::Call => { let target_word = module .functions .iter() - .enumerate() - .find(|(_, (name, _))| *name == r.symbol) - .map(|(j, _)| word_off[j]) + .position(|def| def.name == r.symbol) + .map(|j| word_off[j]) .ok_or(Error::UndefinedSymbol)?; let byte_off = (target_word as i64 - at as i64) * 4; if !(-(128 << 20)..=(128 << 20) - 4).contains(&byte_off) { @@ -280,54 +176,32 @@ pub fn compile_module(module: &Module, relocs: &[Vec]) -> Result = Vec::with_capacity(module.data.len()); + let mut data_syms: Vec = Vec::with_capacity(module.globals.len()); let (mut rodata_off, mut data_off, mut bss_off) = (0usize, 0usize, 0usize); - for d in module.data.iter() { - let a = align_of_data(d.size); - let size = d.size as usize; - match d.kind { - DataKind::Rodata => { - rodata_off = align_forward(rodata_off, a); - data_syms.push(DataSym { - name: d.name.clone(), - kind: d.kind, - off: rodata_off, - size, - bytes: d.bytes.clone(), - relocs: d.relocs.clone(), - }); - rodata_off += size; - } - DataKind::Data => { - data_off = align_forward(data_off, a); - data_syms.push(DataSym { - name: d.name.clone(), - kind: d.kind, - off: data_off, - size, - bytes: d.bytes.clone(), - relocs: d.relocs.clone(), - }); - data_off += size; - } - DataKind::Bss => { - bss_off = align_forward(bss_off, a); - data_syms.push(DataSym { - name: d.name.clone(), - kind: d.kind, - off: bss_off, - size, - bytes: Vec::new(), - relocs: Vec::new(), - }); - bss_off += size; - } - } + for g in module.globals.iter() { + let a = g.effective_align(); + let size = g.size as usize; + let (cursor, bytes, relocs) = match g.kind { + DataKind::Rodata => (&mut rodata_off, g.bytes.clone(), g.relocs.clone()), + DataKind::Data => (&mut data_off, g.bytes.clone(), g.relocs.clone()), + DataKind::Bss => (&mut bss_off, Vec::new(), Vec::new()), + }; + *cursor = align_forward(*cursor, a); + data_syms.push(DataSym { + name: g.name.clone(), + kind: g.kind, + off: *cursor, + size, + align: g.align, + bytes, + relocs, + }); + *cursor += size; } Ok(Linked { code, @@ -339,16 +213,121 @@ pub fn compile_module(module: &Module, relocs: &[Vec]) -> Result Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let v = f.append_inst(block, ty, Opcode::Iconst(value)); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + f + } + + fn function_calling(callee: &str) -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let symbol = f.intern_symbol(callee); + let args = f.intern_values(&[]); + let result = f.append_inst( + block, + ty, + Opcode::Call(Call { + symbol, + args, + is_variadic: false, + num_fixed: 0, + ret_dest: None, + ret_regs: 0, + ret_pieces: [RetPiece::default(); 4], + sret: false, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(result))); + f + } + + fn function_taking_address_of(symbol: &str) -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("ptr").unwrap(); + let block = f.append_block(); + let symbol = f.intern_symbol(symbol); + let v = f.append_inst( + block, + ty, + Opcode::GlobalAddr(GlobalAddr { + symbol, + via_got: false, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + f + } + + #[test] + fn globals_are_laid_out_per_section_with_alignment() { + let mut m = Module::new(); + m.add_data("ro1", vec![1, 2, 3]); + m.add_data("ro2", vec![0; 8]); + m.add_writable("w1", vec![9]); + m.add_writable("w2", vec![0; 4]); + m.add_bss("b1", 3); + m.add_bss("b2", 64); + m.globals[5].align = 64; + let linked = compile_module(&m, &ModelCaps::default()).unwrap(); + let place = |name: &str| { + let d = linked.data.iter().find(|d| d.name == name).unwrap(); + (d.kind, d.off, d.size) + }; + assert_eq!(place("ro1"), (DataKind::Rodata, 0, 3)); + // `ro2` is 8 bytes: aligned to 8 after the 3-byte `ro1`. + assert_eq!(place("ro2"), (DataKind::Rodata, 8, 8)); + assert_eq!(place("w1"), (DataKind::Data, 0, 1)); + assert_eq!(place("w2"), (DataKind::Data, 4, 4)); + assert_eq!(place("b1"), (DataKind::Bss, 0, 3)); + // An explicit alignment overrides the natural (<= 8) one. + assert_eq!(place("b2"), (DataKind::Bss, 64, 64)); + assert!( + linked + .data + .iter() + .find(|d| d.name == "b2") + .unwrap() + .bytes + .is_empty() + ); + } + #[test] - fn data_alignment_covers_scalars() { - assert_eq!(align_of_data(1), 1); - assert_eq!(align_of_data(3), 4); - assert_eq!(align_of_data(8), 8); - assert_eq!(align_of_data(100), 8); + fn global_addr_relocations_carry_forward_unresolved() { + let mut m = Module::new(); + m.add_function("first", function_returning(0)); + m.add_function("addr", function_taking_address_of("table")); + m.add_writable_relocs( + "table", + vec![0; 8], + vec![DataReloc { + offset: 0, + symbol: "first".to_string(), + }], + ); + let linked = compile_module(&m, &ModelCaps::default()).unwrap(); + let start_word = linked.address_of("addr").unwrap() / 4; + let kinds: Vec<_> = linked + .relocs + .iter() + .map(|r| (r.kind, r.symbol.as_str())) + .collect(); + assert!(kinds.contains(&(RelocKind::AdrpPg, "table"))); + assert!(kinds.contains(&(RelocKind::AddPgOff, "table"))); + // Carried-forward offsets are word indices into the whole image. + assert!(linked.relocs.iter().all(|r| r.offset >= start_word)); + assert_eq!(linked.data[0].relocs[0].symbol, "first"); } #[test] @@ -369,15 +348,30 @@ mod tests { #[test] fn compile_module_resolves_call() { let mut m = Module::new(); - m.add_function("callee", vec![0xD65F_03C0]); - m.add_function("caller", vec![encode::bl(0)]); - let relocs = vec![Vec::new(), vec![Reloc::call(0, "callee")]]; - let linked = compile_module(&m, &relocs).unwrap(); + m.add_function("callee", function_returning(42)); + m.add_function("caller", function_calling("callee")); + let linked = compile_module(&m, &ModelCaps::default()).unwrap(); + let caller = linked.address_of("caller").unwrap(); assert_eq!(linked.address_of("callee"), Some(0)); - assert_eq!(linked.address_of("caller"), Some(4)); - // caller word 1 branches back one word: byte offset -4. - assert_eq!(linked.code[1], encode::bl(-4)); + assert!(caller > 0 && caller % 4 == 0); assert!(linked.relocs.is_empty()); + // The caller contains exactly one `bl`, which branches back to word 0. + let bls: Vec = (caller / 4..linked.code.len()) + .filter(|&i| linked.code[i] >> 26 == 0b100101) + .collect(); + assert_eq!(bls.len(), 1); + let at = bls[0]; + assert_eq!(linked.code[at], encode::bl(-((at * 4) as i32))); + } + + #[test] + fn undefined_call_target_is_an_error() { + let mut m = Module::new(); + m.add_function("caller", function_calling("nowhere")); + assert_eq!( + compile_module(&m, &ModelCaps::default()).unwrap_err(), + Error::UndefinedSymbol + ); } #[test] diff --git a/src/target/src/aarch64/lower.rs b/src/target/src/aarch64/lower.rs index 7a58d2f..9bd730d 100644 --- a/src/target/src/aarch64/lower.rs +++ b/src/target/src/aarch64/lower.rs @@ -9,9 +9,10 @@ use crate::regalloc::wimmer::{ }; use alloc::{vec, vec::Vec}; use e::{Cond, FKind, Reg}; -use volt_ir::attribute::{AttrValue, Attribute}; +use volt_ir::attribute::{AttrValue, Attribute, Endianness}; use volt_ir::function::{ - AttrTarget, BinOp, Block, CmpOp, Function, Inst, Jump, Opcode, Ret, Terminator, UnaryOp, Value, + AtomicOrdering, AttrTarget, BinOp, Block, CmpOp, ConvertKind, Function, Inst, Jump, MemFlags, + Opcode, Ret, Terminator, UnaryOp, Value, }; use volt_ir::types::{FloatKind, Type, TypeKind}; @@ -39,9 +40,8 @@ fn quad(f: &Function, v: Value) -> bool { fn vector(f: &Function, v: Value) -> bool { matches!(kind(f, v), TypeKind::Vector(_)) } -fn signed(f: &Function, v: Value) -> bool { - matches!(kind(f,v), TypeKind::Int(i) if i.signed) -} +/// Integer values of fewer than 64 bits are held zero-extended in registers; signedness is a +/// property of the operation, which sign-extends its operands on demand (`signed_operand`). fn bits(f: &Function, v: Value) -> u16 { match kind(f, v) { TypeKind::Int(i) => i.bits, @@ -54,12 +54,24 @@ fn canonical_constant(f: &Function, v: Value, n: u64) -> u64 { let bits = u32::from(bits(f, v)); if bits >= 64 { n - } else if signed(f, v) { - (((n << (64 - bits)) as i64) >> (64 - bits)) as u64 } else { n & ((1u64 << bits) - 1) } } +/// Width in bytes of an `ldar`/`stlr` access to a value of `v`'s type: whole-byte integers up to +/// 64 bits that are a power of two wide, and pointers. Other types have no ordered form. +fn ordered_size(f: &Function, v: Value) -> Option { + match kind(f, v) { + TypeKind::Int(i) if matches!(i.bits, 8 | 16 | 32 | 64) => Some((i.bits / 8) as u8), + TypeKind::Ptr(_) => Some(8), + _ => None, + } +} +/// Whether the access is lowered to `ldar`/`stlr`, which only take a bare `[xn]` address, so no +/// offset may be folded into it. A relaxed access is an ordinary `ldr`/`str` and folds freely. +fn needs_bare_address(mem: &MemFlags) -> bool { + matches!(mem.ordering, Some(o) if o != AtomicOrdering::Relaxed) +} fn wide(f: &Function, v: Value) -> bool { bits(f, v) > 32 } @@ -530,6 +542,15 @@ impl Emitter<'_> { self.frame_imm(false, Reg::X8, base, off)?; Ok((Reg::X8, 0)) } + /// The base register for an `ldar`/`stlr` of `[base + off]`: these have no offset form, so + /// a non-zero `off` is added into the x8 scratch first. + fn bare_base(&mut self, base: Reg, off: usize) -> Result { + if off == 0 { + return Ok(base); + } + self.frame_imm(false, Reg::X8, base, off)?; + Ok(Reg::X8) + } fn loc(&self, v: Value) -> Result { if let Some(&r) = self.alloc.single_regs.get(v.index()) && r != u16::MAX @@ -597,14 +618,14 @@ impl Emitter<'_> { } /// Whether `v = lhs op rhs` (rhs a constant when `imm` is given) is already /// in canonical form, given canonical operands. Values narrower than 64 bits - /// are kept zero- (unsigned) or sign-extended (signed) to the full register. - /// Operations on 32-bit or narrower types run in 32-bit form, which zeroes - /// the upper half, so an unsigned 32-bit result is always canonical; a - /// narrower unsigned result only is if the operation cannot set bits above + /// are kept zero-extended to the full register. Operations on 32-bit or + /// narrower types run in 32-bit form, which zeroes the upper half, so a + /// 32-bit result is always canonical (`SMulh` excepted: it works in 64-bit + /// form); a narrower result only is if the operation cannot set bits above /// the type. fn keeps_canonical_form(&self, v: Value, op: BinOp, imm: Option) -> bool { let n = bits(self.f, v); - if class(self.f, v) != 0 || n >= 64 || signed(self.f, v) || wide(self.f, v) { + if class(self.f, v) != 0 || n >= 64 || wide(self.f, v) || op == BinOp::SMulh { return false; } if n == 32 { @@ -613,7 +634,13 @@ impl Emitter<'_> { let in_range = imm.is_none_or(|k| k >= 0 && (k as u64) >> n == 0); matches!( op, - BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor | BinOp::Shr | BinOp::Div | BinOp::Rem + BinOp::BitAnd + | BinOp::BitOr + | BinOp::BitXor + | BinOp::Shr + | BinOp::UDiv + | BinOp::URem + | BinOp::UMulh ) && in_range } /// `rd = rn op imm` as a single instruction, if the operation has an @@ -622,11 +649,17 @@ impl Emitter<'_> { let is64 = wide(self.f, v); let width = if is64 { 64 } else { 32 }; match op { - BinOp::Shl | BinOp::Shr if (0..width).contains(&imm) => Some(match op { - BinOp::Shl => e::lsl_imm(rd, rn, imm as u32, is64), - _ if signed(self.f, v) => e::asr_imm(rd, rn, imm as u32, is64), - _ => e::lsr_imm(rd, rn, imm as u32, is64), - }), + BinOp::Shl | BinOp::Shr | BinOp::Sar if (0..width).contains(&imm) => { + let n = bits(self.f, v); + Some(match op { + BinOp::Shl => e::lsl_imm(rd, rn, imm as u32, is64), + BinOp::Shr => e::lsr_imm(rd, rn, imm as u32, is64), + _ if n == 32 || n == 64 => e::asr_imm(rd, rn, imm as u32, is64), + // A zero-extended narrow value: extract its bitfield from `imm` up, + // sign-extended from bit `n - 1`. + _ => e::sbfm(rd, rn, imm.min(i64::from(n) - 1) as u8, (n - 1) as u8), + }) + } BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => { let value = if is64 { imm as u64 @@ -646,13 +679,45 @@ impl Emitter<'_> { fn normalize(&mut self, v: Value, r: Reg) { let n = bits(self.f, v); if n < 64 { - self.push(if signed(self.f, v) { - e::sbfm(r, r, 0, (n - 1) as u8) - } else { - e::ubfm(r, r, 0, (n - 1) as u8) - }); + self.push(e::ubfm(r, r, 0, (n - 1) as u8)); } } + /// Whether `signed_operand` has to emit an extension for a value of `v`'s + /// type. Narrow operations read only the low 32 bits of a register, so a + /// 32-bit value needs none unless the whole register is read (`full`). + fn needs_sign_extension(&self, v: Value, full: bool) -> bool { + let n = bits(self.f, v); + n < 64 && (n != 32 || full) + } + /// `r`, which holds `v` zero-extended, as a sign-extended value for an + /// operation that reads its sign: `r` itself when no extension is needed, + /// else `scratch` after an extension into it. + fn signed_operand(&mut self, v: Value, r: Reg, scratch: Reg, full: bool) -> Reg { + if !self.needs_sign_extension(v, full) { + return r; + } + self.push(e::sbfm(scratch, r, 0, (bits(self.f, v) - 1) as u8)); + scratch + } + /// Sets the flags for an integer compare of `a` and `b` (holding `lhs`'s + /// type) and returns the condition that `op` tests. + fn compare_flags(&mut self, op: CmpOp, lhs: Value, a: Reg, b: Reg) -> Result { + let cond = int_cond(op)?; + let (a, b) = if matches!(op, CmpOp::Slt | CmpOp::Sle | CmpOp::Sgt | CmpOp::Sge) { + ( + self.signed_operand(lhs, a, reg(16), false), + self.signed_operand(lhs, b, reg(17), false), + ) + } else { + (a, b) + }; + self.push(if wide(self.f, lhs) { + e::cmp64(a, b) + } else { + e::cmp(a, b) + }); + Ok(cond) + } fn move_one(&mut self, m: Move) -> Result<(), Error> { if m.src == m.dst { return Ok(()); @@ -759,18 +824,15 @@ impl Emitter<'_> { e::str_fp(r, base, off, double(self.f, v)) } } else { - match (load, size, signed(self.f, v)) { - (true, 1, true) => e::ldrsb_off(r, base, off), - (true, 1, false) => e::ldrb_off(r, base, off), - (true, 2, true) => e::ldrsh_off(r, base, off), - (true, 2, false) => e::ldrh_off(r, base, off), - (true, 3..=4, true) => e::ldrsw(r, base, off), - (true, 3..=4, false) => e::ldr_w(r, base, off), - (true, 5..=8, _) => e::ldr_off(r, base, off), - (false, 1, _) => e::strb_off(r, base, off), - (false, 2, _) => e::strh_off(r, base, off), - (false, 3..=4, _) => e::str_w(r, base, off), - (false, 5..=8, _) => e::str_off(r, base, off), + match (load, size) { + (true, 1) => e::ldrb_off(r, base, off), + (true, 2) => e::ldrh_off(r, base, off), + (true, 3..=4) => e::ldr_w(r, base, off), + (true, 5..=8) => e::ldr_off(r, base, off), + (false, 1) => e::strb_off(r, base, off), + (false, 2) => e::strh_off(r, base, off), + (false, 3..=4) => e::str_w(r, base, off), + (false, 5..=8) => e::str_off(r, base, off), _ => return Err(Error::Unsupported), } }; @@ -789,50 +851,67 @@ impl Emitter<'_> { b: Reg, ) -> Result<(), Error> { let sf = if wide(self.f, v) { 1 << 31 } else { 0 }; - let si = signed(self.f, v); let w = match op { BinOp::Add => e::add(rd, a, b), BinOp::Sub => e::sub(rd, a, b), BinOp::Mul => e::mul(rd, a, b), - BinOp::Mulh => { - if bits(self.f, v) != 64 { + BinOp::UMulh | BinOp::SMulh => { + let n = bits(self.f, v); + if n == 64 { + self.push(if op == BinOp::SMulh { + e::smulh(rd, a, b) + } else { + e::umulh(rd, a, b) + }); + return Ok(()); + } + // The product of two values of up to 32 bits fits a register. + if n > 32 { return Err(Error::Unsupported); } - if si { - e::smulh(rd, a, b) + if op == BinOp::SMulh { + let a = self.signed_operand(v, a, reg(16), true); + let b = self.signed_operand(v, b, reg(17), true); + self.push(e::mul64(rd, a, b)); + self.push(e::asr_imm(rd, rd, u32::from(n), true)); } else { - e::umulh(rd, a, b) + self.push(e::mul64(rd, a, b)); + self.push(e::lsr_imm(rd, rd, u32::from(n), true)); } + return Ok(()); } - BinOp::Div => { - if si { - e::sdiv(rd, a, b) - } else { - e::udiv(rd, a, b) - } + BinOp::UDiv => e::udiv(rd, a, b), + BinOp::SDiv => { + let a = self.signed_operand(v, a, reg(16), false); + let b = self.signed_operand(v, b, reg(17), false); + e::sdiv(rd, a, b) } - BinOp::Rem => { - self.push( - sf | if si { - e::sdiv(reg(16), a, b) - } else { - e::udiv(reg(16), a, b) - }, - ); + BinOp::URem => { + self.push(sf | e::udiv(reg(16), a, b)); self.push(sf | e::msub(rd, reg(16), b, a)); return Ok(()); } + BinOp::SRem => { + // Extended operands live in x16/x17, so the quotient can use `rd`. + let extend = self.needs_sign_extension(v, false); + let a = self.signed_operand(v, a, reg(16), false); + let b = self.signed_operand(v, b, reg(17), false); + let q = if extend { rd } else { reg(16) }; + self.push(sf | e::sdiv(q, a, b)); + self.push(sf | e::msub(rd, q, b, a)); + return Ok(()); + } BinOp::BitAnd => e::andr(rd, a, b), BinOp::BitOr => e::orr(rd, a, b), BinOp::BitXor => e::eor(rd, a, b), BinOp::Shl => e::lslv(rd, a, b), - BinOp::Shr => { - if si { - e::asrv(rd, a, b) - } else { - e::lsrv(rd, a, b) - } + BinOp::Shr => e::lsrv(rd, a, b), + BinOp::Sar => { + let a = self.signed_operand(v, a, reg(16), false); + e::asrv(rd, a, b) } + // Floating-point only. + BinOp::Div | BinOp::Rem => return Err(Error::Unsupported), }; self.push(sf | w); Ok(()) @@ -915,7 +994,7 @@ impl Emitter<'_> { } self.finish(v, rd) } - fn convert(&mut self, v: Value, src: Value) -> Result<(), Error> { + fn convert(&mut self, v: Value, src: Value, kind: ConvertKind) -> Result<(), Error> { if quad(self.f, v) || quad(self.f, src) || vector(self.f, v) || vector(self.f, src) { return Err(Error::Unsupported); } @@ -923,25 +1002,43 @@ impl Emitter<'_> { let dc = class(self.f, v); let a = self.read(src, reg(if sc == 1 { 24 } else { 13 }))?; let rd = self.result(v)?; - match (sc, dc) { - (0, 1) => { - self.push(e::cvt_int_to_float( - rd, - a, - fkind(self.f, v, self.caps), - signed(self.f, src), - )); + match kind { + ConvertKind::SiToFp | ConvertKind::UiToFp => { + if (sc, dc) != (0, 1) { + return Err(Error::Unsupported); + } + let si = kind == ConvertKind::SiToFp; + let a = if si { + self.signed_operand(src, a, reg(16), false) + } else { + a + }; + let sf = if wide(self.f, src) { 1 << 31 } else { 0 }; + self.push(sf | e::cvt_int_to_float(rd, a, fkind(self.f, v, self.caps), si)); if half(self.f, v) && !self.caps.fp16 { self.half_round(rd); } } - (1, 0) => self.push(e::cvt_float_to_int( - rd, - a, - fkind(self.f, src, self.caps), - signed(self.f, v), - )), - (1, 1) => { + ConvertKind::FpToSi | ConvertKind::FpToUi => { + if (sc, dc) != (1, 0) { + return Err(Error::Unsupported); + } + // A destination narrower than 32 bits keeps the low bits of the 32-bit + // saturating conversion (`finish` truncates). + let sf = if wide(self.f, v) { 1 << 31 } else { 0 }; + self.push( + sf | e::cvt_float_to_int( + rd, + a, + fkind(self.f, src, self.caps), + kind == ConvertKind::FpToSi, + ), + ); + } + ConvertKind::FpResize => { + if (sc, dc) != (1, 1) { + return Err(Error::Unsupported); + } if self.caps.fp16 && (half(self.f, src) || half(self.f, v)) { self.push(if half(self.f, src) && half(self.f, v) { e::fmov_reg(rd, a) @@ -969,16 +1066,27 @@ impl Emitter<'_> { self.push(e::fcvt(rd, a, double(self.f, v))); } } - _ => { + ConvertKind::Trunc | ConvertKind::Zext | ConvertKind::Sext => { + if (sc, dc) != (0, 0) { + return Err(Error::Unsupported); + } let n = bits(self.f, src); - if bits(self.f, v) > n && n < 64 { - self.push(if signed(self.f, src) { - e::sbfm(rd, a, 0, (n - 1) as u8) - } else { - e::ubfm(rd, a, 0, (n - 1) as u8) - }); + let m = bits(self.f, v); + if kind == ConvertKind::Sext && m > n && n < 64 { + let word = e::sbfm(rd, a, 0, (n - 1) as u8); + if m == 32 { + // The 32-bit form clears the upper half: already canonical. + self.push(word & !(1 << 31 | 1 << 22)); + return self.finish_normal(v, rd); + } + self.push(word); } else { + // A zero-extension of a canonical value, a truncation (`finish` + // clears the bits above `m`) or a same-width copy. self.push(e::mov(rd, a)); + if kind == ConvertKind::Zext { + return self.finish_normal(v, rd); + } } } } @@ -1132,19 +1240,12 @@ impl Emitter<'_> { if let Fusion::Compare { lhs, rhs, op } = fusion { let a = self.read(lhs, reg(13))?; let b = self.read(rhs, reg(14))?; - self.push(if wide(self.f, lhs) { - e::cmp64(a, b) - } else { - e::cmp(a, b) - }); + let cond = self.compare_flags(op, lhs, a, b)?; let Opcode::If(cf) = self.f.opcode_ref(inst) else { return Err(Error::Unsupported); }; let other = self.local_label(); - self.branch( - other, - e::bcc(e::invert(int_cond(op, signed(self.f, lhs))), 0), - ); + self.branch(other, e::bcc(e::invert(cond), 0)); self.edge(cf.then)?; self.bind(other); self.edge(cf.else_)?; @@ -1222,7 +1323,7 @@ impl Emitter<'_> { } else if let Some(word) = self.immediate_form(v, a.op, rd, r, a.imm) { self.push(word); } else { - self.constant(reg(14), a.imm as u64); + self.constant(reg(14), canonical_constant(self.f, v, a.imm as u64)); self.integer_binary(v, a.op, rd, r, reg(14))?; } if self.keeps_canonical_form(v, a.op, Some(a.imm)) { @@ -1247,6 +1348,7 @@ impl Emitter<'_> { CmpOp::Le => e::fcmge_vec(rd, b, a), CmpOp::Gt => e::fcmgt_vec(rd, a, b), CmpOp::Ge => e::fcmge_vec(rd, a, b), + _ => return Err(Error::Unsupported), }); if c.op == CmpOp::Ne { self.push(e::mvn_vec(rd, rd)); @@ -1255,15 +1357,12 @@ impl Emitter<'_> { if quad(self.f, c.lhs) { return Err(Error::Unsupported); } + let cond = float_cond(c.op)?; self.push(e::fcmp(a, b, fkind(self.f, c.lhs, self.caps))); - self.push(e::cset(rd, float_cond(c.op))); + self.push(e::cset(rd, cond)); } else { - self.push(if wide(self.f, c.lhs) { - e::cmp64(a, b) - } else { - e::cmp(a, b) - }); - self.push(e::cset(rd, int_cond(c.op, signed(self.f, c.lhs)))); + let cond = self.compare_flags(c.op, c.lhs, a, b)?; + self.push(e::cset(rd, cond)); } // `cset` already yields 0 or 1. if !fp && !vector(self.f, c.lhs) && bits(self.f, v) == 1 { @@ -1297,15 +1396,17 @@ impl Emitter<'_> { e::csel(rd, a, b, Cond::Ne) | if wide(self.f, v) { 1 << 31 } else { 0 } }); } - // `csel` copies one of two canonical operands, so an unsigned - // result needs no extension. - if class(self.f, v) == 0 && !signed(self.f, v) && !vector(self.f, v) { + // `csel` copies one of two canonical operands, so the result + // needs no extension. + if class(self.f, v) == 0 && !vector(self.f, v) { self.finish_normal(v, rd)?; } else { self.finish(v, rd)?; } } - Opcode::Convert(c) => self.convert(result.ok_or(Error::Unsupported)?, c.value)?, + Opcode::Convert(c) => { + self.convert(result.ok_or(Error::Unsupported)?, c.value, c.kind)? + } Opcode::Unary(u) => { let v = result.ok_or(Error::Unsupported)?; let rd = self.result(v)?; @@ -1370,25 +1471,73 @@ impl Emitter<'_> { self.finish(v, rd)?; } Opcode::Load(l) => { + if l.mem.endian == Endianness::Big { + return Err(Error::Unsupported); + } let v = result.ok_or(Error::Unsupported)?; + let size = ordered_size(self.f, v); + let acquire = match l.mem.ordering { + None => false, + // Relaxed: an ordinary, naturally aligned load is single-copy atomic. + Some(AtomicOrdering::Relaxed) => { + if size.is_none() { + return Err(Error::Unsupported); + } + false + } + Some(AtomicOrdering::Acquire | AtomicOrdering::SeqCst) => true, + Some(AtomicOrdering::Release | AtomicOrdering::AcqRel) => { + return Err(Error::Unsupported); + } + }; let a = self.read(l.ptr, reg(13))?; let rd = self.result(v)?; - self.memory(true, v, rd, a, self.fold.off_of(inst) as usize)?; - // `ldrb`/`ldrh`/`ldr w` already zero-extend a whole-byte unsigned value. + if acquire { + let size = size.ok_or(Error::Unsupported)?; + let base = self.bare_base(a, self.fold.off_of(inst) as usize)?; + self.push(e::ldar(rd, base, size)); + } else { + self.memory(true, v, rd, a, self.fold.off_of(inst) as usize)?; + } + // `ldrb`/`ldrh`/`ldr w`/`ldar*` already zero-extend a whole-byte value. let n = bits(self.f, v); - if class(self.f, v) == 0 && !signed(self.f, v) && matches!(n, 8 | 16 | 32 | 64) { + if class(self.f, v) == 0 && matches!(n, 8 | 16 | 32 | 64) { self.finish_normal(v, rd)?; } else { self.finish(v, rd)?; } } Opcode::Store(s) => { + if s.mem.endian == Endianness::Big { + return Err(Error::Unsupported); + } + let size = ordered_size(self.f, s.value); + let release = match s.mem.ordering { + None => false, + // Relaxed: an ordinary, naturally aligned store is single-copy atomic. + Some(AtomicOrdering::Relaxed) => { + if size.is_none() { + return Err(Error::Unsupported); + } + false + } + Some(AtomicOrdering::Release | AtomicOrdering::SeqCst) => true, + Some(AtomicOrdering::Acquire | AtomicOrdering::AcqRel) => { + return Err(Error::Unsupported); + } + }; let a = self.read( s.value, reg(if class(self.f, s.value) == 1 { 24 } else { 13 }), )?; let p = self.read(s.ptr, reg(14))?; - self.memory(false, s.value, a, p, self.fold.off_of(inst) as usize)?; + if release { + let size = size.ok_or(Error::Unsupported)?; + let base = self.bare_base(p, self.fold.off_of(inst) as usize)?; + self.push(e::stlr(a, base, size)); + } else { + self.memory(false, s.value, a, p, self.fold.off_of(inst) as usize)?; + } } Opcode::Prefetch(p) => { let a = self.read(p.ptr, reg(13))?; @@ -1464,7 +1613,12 @@ impl Emitter<'_> { } else { e::umov_lane(rd, a, x.index as u8) }); - self.finish(v, rd)?; + if fp { + self.finish(v, rd)?; + } else { + // `umov` to a w register zero-extends the 32-bit lane. + self.finish_normal(v, rd)?; + } } Opcode::StructNew(s) => { let v = result.ok_or(Error::Unsupported)?; @@ -1520,12 +1674,10 @@ impl Emitter<'_> { let b = self.read(d.a, reg(25))?; let c = self.read(d.b, reg(26))?; self.push(e::mov_vec(reg(27), a)); - let si = match kind(self.f, d.a) { - TypeKind::Vector(x) => { - matches!(self.f.types.type_kind(x.elem),TypeKind::Int(i) if i.signed) - } - _ => return Err(Error::Unsupported), - }; + if !vector(self.f, d.a) { + return Err(Error::Unsupported); + } + let si = d.signed; self.push(if si { e::sdot(reg(27), b, c) } else { @@ -1601,49 +1753,33 @@ impl Emitter<'_> { Ok(()) } } -fn int_cond(op: CmpOp, si: bool) -> Cond { - match op { +/// The condition code that holds after comparing the operands of an integer +/// (or pointer) `op`; the signed forms need sign-extended operands. +fn int_cond(op: CmpOp) -> Result { + Ok(match op { CmpOp::Eq => Cond::Eq, CmpOp::Ne => Cond::Ne, - CmpOp::Lt => { - if si { - Cond::Lt - } else { - Cond::Lo - } - } - CmpOp::Le => { - if si { - Cond::Le - } else { - Cond::Ls - } - } - CmpOp::Gt => { - if si { - Cond::Gt - } else { - Cond::Hi - } - } - CmpOp::Ge => { - if si { - Cond::Ge - } else { - Cond::Hs - } - } - } + CmpOp::Slt => Cond::Lt, + CmpOp::Sle => Cond::Le, + CmpOp::Sgt => Cond::Gt, + CmpOp::Sge => Cond::Ge, + CmpOp::Ult => Cond::Lo, + CmpOp::Ule => Cond::Ls, + CmpOp::Ugt => Cond::Hi, + CmpOp::Uge => Cond::Hs, + CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge => return Err(Error::Unsupported), + }) } -fn float_cond(op: CmpOp) -> Cond { - match op { +fn float_cond(op: CmpOp) -> Result { + Ok(match op { CmpOp::Eq => Cond::Eq, CmpOp::Ne => Cond::Ne, CmpOp::Lt => Cond::Mi, CmpOp::Le => Cond::Ls, CmpOp::Gt => Cond::Gt, CmpOp::Ge => Cond::Ge, - } + _ => return Err(Error::Unsupported), + }) } /// True for a function made only of scalar integer, boolean and pointer @@ -1671,6 +1807,14 @@ fn is_plain_integer(f: &Function) -> bool { ) { return false; } + let mem = match f.opcode_ref(*i) { + Opcode::Load(l) => Some(&l.mem), + Opcode::Store(s) => Some(&s.mem), + _ => None, + }; + if mem.is_some_and(|m| !m.is_plain()) { + return false; + } } } (0..f.value_count()).all(|v| match kind(f, Value(v as u32)) { @@ -1709,13 +1853,16 @@ pub(super) fn compile_owned( _ => false, }); volt_ir::softfp::lower(&mut f); + // Legalize rewrites byte-swapped and atomic float accesses into forms selected below; + // ordered int/pointer accesses become `ldar`/`stlr` (or plain accesses when relaxed). + volt_ir::legalize::legalize(&mut f); volt_ir::critical_edge::split_critical_edges(&mut f); volt_ir::reachable::neutralize_unreachable(&mut f); } let fold = crate::regalloc::addrfold::analyze(&f, |f, i| { let (p, v) = match f.opcode_ref(i) { - Opcode::Load(l) => (l.ptr, f.inst_result(i)?), - Opcode::Store(s) => (s.ptr, s.value), + Opcode::Load(l) if !needs_bare_address(&l.mem) => (l.ptr, f.inst_result(i)?), + Opcode::Store(s) if !needs_bare_address(&s.mem) => (s.ptr, s.value), _ => return None, }; let add = match f.opcode_ref(f.defining_inst(p)?) { diff --git a/src/target/src/aarch64/mod.rs b/src/target/src/aarch64/mod.rs index 2ed3b76..e149dd7 100644 --- a/src/target/src/aarch64/mod.rs +++ b/src/target/src/aarch64/mod.rs @@ -1,4 +1,5 @@ //! AArch64 encoding, Function-based native selection, object linking and JIT. +pub mod decode; pub mod disasm; pub mod encode; pub mod isel; @@ -13,3 +14,4 @@ pub mod jit; pub mod link; pub mod object; pub mod peephole; +pub mod simd_struct; diff --git a/src/target/src/aarch64/object.rs b/src/target/src/aarch64/object.rs index cd1da2c..3421e31 100644 --- a/src/target/src/aarch64/object.rs +++ b/src/target/src/aarch64/object.rs @@ -274,8 +274,8 @@ fn reloc_flags(reloc_type: RelocType) -> object::RelocationFlags { /// are emitted as undefined extern symbols. pub fn write_module_object(linked: &super::link::Linked) -> Vec { use super::isel::RelocKind; - use super::link::DataKind; use alloc::collections::BTreeMap; + use volt_ir::module::DataKind; let mut symbols: Vec = Vec::new(); let mut id_of_name: BTreeMap<&str, u32> = BTreeMap::new(); @@ -298,15 +298,17 @@ pub fn write_module_object(linked: &super::link::Linked) -> Vec { for d in linked.data.iter() { let section = match d.kind { DataKind::Rodata => { + rodata_bytes.resize(d.off, 0); rodata_bytes.extend_from_slice(&d.bytes); SectionKind::Rodata } DataKind::Data => { + data_bytes.resize(d.off, 0); data_bytes.extend_from_slice(&d.bytes); SectionKind::Data } DataKind::Bss => { - bss_size += d.size as u64; + bss_size = (d.off + d.size) as u64; SectionKind::Bss } }; @@ -370,7 +372,7 @@ pub fn write_module_object(linked: &super::link::Linked) -> Vec { for r in d.relocs.iter() { data_relocs.push(DataRelocEntry { section, - offset: (d.off + r.off) as u64, + offset: (d.off + r.offset) as u64, symbol: id_of_name[r.symbol.as_str()], addend: 0, }); @@ -391,9 +393,12 @@ pub fn write_module_object(linked: &super::link::Linked) -> Vec { #[cfg(test)] mod tests { use super::*; - use crate::aarch64::link::{DataReloc, Linked}; + use crate::aarch64::isel::ModelCaps; + use crate::aarch64::link::{Linked, compile_module}; use alloc::vec; - use object::read::{Object as _, ObjectSymbol as _}; + use object::read::{Object as _, ObjectSection as _, ObjectSymbol as _}; + use volt_ir::function::{Function, Opcode, Ret, Terminator}; + use volt_ir::module::{DataReloc, Module}; #[test] fn elf_round_trip_aarch64() { @@ -474,9 +479,47 @@ mod tests { assert_eq!(obj.architecture(), object::Architecture::Aarch64); assert!(obj.sections().count() >= 2); assert!(obj.symbols().count() >= 2, "defined + extern symbols"); - let _ = DataReloc { - off: 0, - symbol: String::new(), - }; + } + + #[test] + fn elf_from_an_ir_module_with_globals() { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let v = f.append_inst(block, ty, Opcode::Iconst(5)); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + let mut m = Module::new(); + m.add_function("five", f); + m.add_data("ro", vec![1, 2, 3, 4]); + m.add_data("ro8", vec![0; 8]); + m.add_writable_relocs( + "table", + vec![0; 16], + vec![DataReloc { + offset: 8, + symbol: "five".to_string(), + }], + ); + m.add_bss("scratch", 32); + let linked = compile_module(&m, &ModelCaps::default()).unwrap(); + let bytes = write_module_object(&linked); + let obj = object::read::File::parse(&bytes[..]).expect("module ELF parses"); + assert_eq!(obj.architecture(), object::Architecture::Aarch64); + let defined: Vec = obj + .symbols() + .filter(|s| s.is_definition()) + .filter_map(|s| s.name().ok().map(|n| n.to_string())) + .collect(); + for want in ["five", "ro", "table", "scratch"] { + assert!(defined.iter().any(|n| n == want), "symbol {want} defined"); + } + let data = obj.section_by_name(".data").expect(".data section"); + assert_eq!(data.data().unwrap().len(), 16); + // `ro8` is placed at offset 8, so `ro` is padded to it. + let rodata = obj.section_by_name(".data.rel.ro").expect("rodata section"); + assert_eq!(rodata.data().unwrap().len(), 16); + let relocs: Vec<_> = data.relocations().collect(); + assert_eq!(relocs.len(), 1); + assert_eq!(relocs[0].0, 8, "ABS64 slot offset within .data"); } } diff --git a/src/target/src/aarch64/simd_struct.rs b/src/target/src/aarch64/simd_struct.rs new file mode 100644 index 0000000..4f516c0 --- /dev/null +++ b/src/target/src/aarch64/simd_struct.rs @@ -0,0 +1,51 @@ +//! SIMD structure loads and stores: `LD1`..`LD4`, `ST1`..`ST4`, their single-lane +//! forms and the replicating `LD1R`..`LD4R`. +//! +//! Decode describes one in a [`StructDesc`]. The emulator copies the descriptor into +//! its vCPU state and runs the access against its own memory, one element at a time. + +/// Which elements of the registers a structure access touches. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum Shape { + /// Whole vectors: `ld1 {v0.4s - v3.4s}` and the interleaved `ld2`..`ld4`. + #[default] + Multiple, + /// One element of each register: `ld1 {v0.s}[1]`. Other lanes are kept. + Lane, + /// One element broadcast to every lane: `ld1r`. Loads only. + Replicate, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C)] +pub struct StructDesc { + pub store: bool, + pub shape: Shape, + /// Registers touched, starting at `rt` and wrapping from v31 to v0. + pub registers: u8, + /// `ld2`..`ld4`: consecutive memory elements alternate between the registers. + /// Otherwise (`ld1`) each register is a contiguous block. + pub interleaved: bool, + /// Element size in bytes: 1, 2, 4 or 8. + pub esize: u8, + /// Element index of a [`Shape::Lane`] access. + pub lane: u8, + /// A 128-bit vector (otherwise 64-bit, with the upper half zeroed on loads). + pub q: bool, + pub rt: u8, +} + +impl StructDesc { + pub fn vector_bytes(&self) -> u8 { + if self.q { 16 } else { 8 } + } + /// Bytes of memory the access spans; the post-index immediate advances by this. + pub fn bytes(&self) -> u64 { + let registers = u64::from(self.registers); + match self.shape { + Shape::Multiple => registers * u64::from(self.vector_bytes()), + Shape::Lane | Shape::Replicate => registers * u64::from(self.esize), + } + } +} diff --git a/src/target/src/lib.rs b/src/target/src/lib.rs index ca51096..3f25aeb 100644 --- a/src/target/src/lib.rs +++ b/src/target/src/lib.rs @@ -12,24 +12,3 @@ pub mod x86_64; pub mod native; pub mod regalloc; - -use core::fmt::Debug; - -/// Failure to decode a guest instruction. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct DecodeError { - /// Original instruction bits, useful for diagnostics and guest exceptions. - pub word: W, -} - -/// Architecture-specific decoder. `W` is the instruction word representation; -/// variable-length ISAs can use a byte slice in their own decoder implementation. -pub trait Decoder { - /// Encoded instruction word. - type Word: Copy + Debug; - /// Decoded architecture-specific instruction. - type Instruction: Debug; - - /// Decode one instruction or reject the encoding. - fn decode(&self, word: Self::Word) -> Result>; -} diff --git a/src/target/src/native.rs b/src/target/src/native.rs index dae8301..7ac7eaa 100644 --- a/src/target/src/native.rs +++ b/src/target/src/native.rs @@ -23,10 +23,7 @@ pub use module::map_linked_aarch64; all(target_arch = "x86_64", feature = "x86_64"), all(target_arch = "aarch64", feature = "aarch64") ))] -pub use module::{ - DataKind, DataReloc, Error as ModuleError, JittedModule, ModuleData, ModuleFunction, - jit_module, jit_module_data, -}; +pub use module::{Error as ModuleError, JittedModule, jit_module}; #[derive(Debug)] pub enum Error { diff --git a/src/target/src/native/module.rs b/src/target/src/native/module.rs index 37a3072..df2b60d 100644 --- a/src/target/src/native/module.rs +++ b/src/target/src/native/module.rs @@ -3,7 +3,7 @@ use super::{Error as NativeError, JittedFunction, sync_icache}; use core::{fmt, ptr::NonNull}; -use volt_ir::function::Function; +use volt_ir::module::{DataKind, Global, Module}; #[cfg(target_arch = "aarch64")] use crate::aarch64::isel; @@ -48,40 +48,14 @@ impl From for Error { } } -pub struct ModuleFunction<'a> { - pub name: &'a str, - pub function: &'a Function, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum DataKind { - Rodata, - Data, - Bss, -} -impl DataKind { - fn section(self) -> usize { - match self { - Self::Rodata => 0, - Self::Data => 1, - Self::Bss => 2, - } +fn section_index(kind: DataKind) -> usize { + match kind { + DataKind::Rodata => 0, + DataKind::Data => 1, + DataKind::Bss => 2, } } -pub struct DataReloc<'a> { - pub offset: usize, - pub symbol: &'a str, -} - -pub struct ModuleData<'a> { - pub name: &'a str, - pub bytes: &'a [u8], - pub kind: DataKind, - pub size: usize, - pub relocs: &'a [DataReloc<'a>], -} - /// One contiguous image. Code is RX, constants R, mutable globals and BSS RW. pub struct JittedModule { image: JittedFunction, @@ -133,32 +107,27 @@ fn symbol_offset( .ok_or_else(|| Error::UndefinedSymbol(name.into())) } -pub fn jit_module(functions: &[ModuleFunction<'_>]) -> Result { - jit_module_data(functions, &[]) -} - -pub fn jit_module_data( - functions: &[ModuleFunction<'_>], - data: &[ModuleData<'_>], -) -> Result { +/// Compile `module` for the host, map it as one image, and resolve every +/// intra-module call, `global_addr`, and data relocation. +pub fn jit_module(module: &Module) -> Result { #[cfg(not(target_os = "linux"))] { - let _ = (functions, data); + let _ = module; return Err(NativeError::UnsupportedHost.into()); } #[cfg(target_os = "linux")] { - let mut compiled = Vec::with_capacity(functions.len()); - let mut symbols = Vec::with_capacity(functions.len()); + let mut compiled = Vec::with_capacity(module.functions.len()); + let mut symbols = Vec::with_capacity(module.functions.len()); let mut code_len = 0usize; - for function in functions { + for def in &module.functions { #[cfg(target_arch = "aarch64")] - let part = isel::compile_function(function.function, &isel::ModelCaps::default()) + let part = isel::compile_function(&def.function, &isel::ModelCaps::default()) .map_err(|e| Error::Compile(Box::new(e)))?; #[cfg(target_arch = "x86_64")] let part = - isel::compile_object(function.function).map_err(|e| Error::Compile(Box::new(e)))?; - symbols.push((function.name.to_owned(), code_len)); + isel::compile_object(&def.function).map_err(|e| Error::Compile(Box::new(e)))?; + symbols.push((def.name.clone(), code_len)); #[cfg(target_arch = "aarch64")] let length = part .code @@ -179,7 +148,7 @@ pub fn jit_module_data( map_module( code_len, symbols, - data, + &module.globals, |memory, base, symbols, data_symbols| { #[cfg(target_arch = "x86_64")] memory[..code_len].fill(0x90); @@ -293,7 +262,7 @@ pub fn jit_module_data( fn map_module( code_len: usize, symbols: Vec<(String, usize)>, - data: &[ModuleData<'_>], + data: &[Global], fill_code: impl FnOnce( &mut [u8], usize, @@ -308,47 +277,42 @@ fn map_module( for name in symbols .iter() .map(|(name, _)| name.as_str()) - .chain(data.iter().map(|d| d.name)) + .chain(data.iter().map(|d| d.name.as_str())) { if !names.insert(name) { return Err(Error::DuplicateSymbol(name.into())); } } + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page <= 0 { + return Err(NativeError::Mapping(std::io::Error::last_os_error()).into()); + } + let page = page as usize; let mut extents = [0usize; 3]; let mut placements = Vec::with_capacity(data.len()); for object in data { - if (object.kind != DataKind::Bss && object.size != object.bytes.len()) + if (object.kind != DataKind::Bss && object.size != object.bytes.len() as u64) || (object.kind == DataKind::Bss && !object.bytes.is_empty()) + || (object.align != 0 && !object.align.is_power_of_two()) { return Err(Error::InvalidData); } - let section = object.kind.section(); - let natural = object - .size - .max(1) - .checked_next_power_of_two() - .unwrap_or(8) - .min(8); - let offset = align(extents[section], natural)?; - extents[section] = offset - .checked_add(object.size) - .ok_or(Error::AddressOverflow)?; - for reloc in object.relocs { - if reloc - .offset - .checked_add(8) - .is_none_or(|end| end > object.size) - { + // Each section starts page-aligned, so no stricter alignment is honorable. + let alignment = object.effective_align(); + if alignment > page { + return Err(Error::InvalidData); + } + let size = usize::try_from(object.size).map_err(|_| Error::AddressOverflow)?; + let section = section_index(object.kind); + let offset = align(extents[section], alignment)?; + extents[section] = offset.checked_add(size).ok_or(Error::AddressOverflow)?; + for reloc in &object.relocs { + if reloc.offset.checked_add(8).is_none_or(|end| end > size) { return Err(Error::InvalidRelocation); } } placements.push(offset); } - let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; - if page <= 0 { - return Err(NativeError::Mapping(std::io::Error::last_os_error()).into()); - } - let page = page as usize; let mut starts = [0usize; 3]; let mut mapped_len = align(code_len, page)?; for section in 0..3 { @@ -384,14 +348,14 @@ fn map_module( let memory = unsafe { core::slice::from_raw_parts_mut(address.as_ptr(), mapped_len) }; let mut data_symbols = Vec::with_capacity(data.len()); for (object, placement) in data.iter().zip(&placements) { - let offset = starts[object.kind.section()] + placement; - memory[offset..offset + object.bytes.len()].copy_from_slice(object.bytes); - data_symbols.push((object.name.to_owned(), offset)); + let offset = starts[section_index(object.kind)] + placement; + memory[offset..offset + object.bytes.len()].copy_from_slice(&object.bytes); + data_symbols.push((object.name.clone(), offset)); } let base = address.as_ptr() as usize; for (object, (_, offset)) in data.iter().zip(&data_symbols) { - for reloc in object.relocs { - let target = base + symbol_offset(&symbols, &data_symbols, reloc.symbol)?; + for reloc in &object.relocs { + let target = base + symbol_offset(&symbols, &data_symbols, &reloc.symbol)?; let site = offset + reloc.offset; memory[site..site + 8].copy_from_slice(&(target as u64).to_le_bytes()); } @@ -447,34 +411,16 @@ pub fn map_linked_aarch64(linked: &crate::aarch64::link::Linked) -> Result> = linked + let objects: Vec = linked .data .iter() - .map(|object| { - object - .relocs - .iter() - .map(|r| DataReloc { - offset: r.off, - symbol: r.symbol.as_str(), - }) - .collect() - }) - .collect(); - let objects: Vec<_> = linked - .data - .iter() - .zip(&data_relocs) - .map(|(object, relocs)| ModuleData { - name: &object.name, - bytes: &object.bytes, - kind: match object.kind { - link::DataKind::Rodata => DataKind::Rodata, - link::DataKind::Data => DataKind::Data, - link::DataKind::Bss => DataKind::Bss, - }, - size: object.size, - relocs, + .map(|object| Global { + name: object.name.clone(), + kind: object.kind, + bytes: object.bytes.clone(), + size: object.size as u64, + align: object.align, + relocs: object.relocs.clone(), }) .collect(); map_module( @@ -520,3 +466,197 @@ pub fn map_linked_aarch64(linked: &crate::aarch64::link::Linked) -> Result u64; + type NullaryFn = extern "C" fn() -> u64; + + /// `name(x) = x + 7`. + fn add_seven() -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let x = f.append_block_param(block, ty); + let r = f.append_inst( + block, + ty, + Opcode::ArithImm(ArithImm { + op: BinOp::Add, + lhs: x, + imm: 7, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(r))); + f + } + + /// `name(x) = callee(x) * 3`. + fn triple_of(callee: &str) -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let x = f.append_block_param(block, ty); + let c = f.append_call(block, ty, callee, &[x]); + let r = f.append_inst( + block, + ty, + Opcode::ArithImm(ArithImm { + op: BinOp::Mul, + lhs: c, + imm: 3, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(r))); + f + } + + /// `name() = *global`, a u64 load through `global_addr`. + fn load_global(global: &str) -> Function { + let mut f = Function::new(); + let u64_t = f.types.parse_type("i64").unwrap(); + let ptr_t = f.types.parse_type("ptr").unwrap(); + let block = f.append_block(); + let symbol = f.intern_symbol(global); + let addr = f.append_inst( + block, + ptr_t, + Opcode::GlobalAddr(GlobalAddr { + symbol, + via_got: false, + }), + ); + let v = f.append_load(block, u64_t, addr, MemFlags::new()); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + f + } + + fn sample_module() -> Module { + let mut m = Module::new(); + m.add_function("caller", triple_of("callee")); + m.add_function("callee", add_seven()); + m.add_function("magic", load_global("constant")); + m.add_data("constant", 0x1122_3344_5566_7788u64.to_le_bytes().to_vec()); + m.add_writable_relocs( + "table", + vec![0; 16], + vec![DataReloc { + offset: 8, + symbol: "callee".to_string(), + }], + ); + m.add_bss("scratch", 24); + m + } + + #[test] + fn functions_call_each_other_and_data_relocates_to_a_function() { + let jit = match jit_module(&sample_module()) { + Ok(jit) => jit, + Err(e) => panic!("jit_module failed: {e}"), + }; + // SAFETY: the signatures match the IR functions above and the host ISA + // is the one this module was compiled for. + unsafe { + let caller: UnaryFn = jit.entry("caller").unwrap(); + let callee: UnaryFn = jit.entry("callee").unwrap(); + assert_eq!(callee(5), 12); + assert_eq!(caller(5), 36); + let magic: NullaryFn = jit.entry("magic").unwrap(); + assert_eq!(magic(), 0x1122_3344_5566_7788); + + // The data relocation holds the absolute address of `callee`. + let table = jit.data_addr("table").unwrap(); + assert_eq!( + core::slice::from_raw_parts(table, 8), + &[0; 8], + "slot without a relocation stays as initialized" + ); + let slot = table.add(8).cast::().read_unaligned(); + assert_eq!(slot, callee as usize as u64); + let through_table: UnaryFn = core::mem::transmute::(slot as usize); + assert_eq!(through_table(1), 8); + + // Writable globals are really writable and BSS starts zeroed. + let scratch = jit.data_addr("scratch").unwrap(); + assert_eq!(core::slice::from_raw_parts(scratch, 24), &[0; 24]); + scratch.write(0xAB); + assert_eq!(scratch.read(), 0xAB); + } + assert!(jit.data_addr("callee").is_none(), "functions are not data"); + assert!(jit.data_addr("nothing").is_none()); + } + + #[test] + fn rejects_undefined_data_relocation_and_duplicate_names() { + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_data_relocs( + "t", + vec![0; 8], + vec![DataReloc { + offset: 0, + symbol: "missing".to_string(), + }], + ); + assert!(matches!( + jit_module(&m), + Err(Error::UndefinedSymbol(name)) if name == "missing" + )); + + let mut m = Module::new(); + m.add_function("same", add_seven()); + m.add_data("same", vec![0; 8]); + assert!(matches!( + jit_module(&m), + Err(Error::DuplicateSymbol(name)) if name == "same" + )); + } + + #[test] + fn rejects_malformed_globals() { + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_data("d", vec![0; 4]); + m.globals[0].size = 8; + assert!(matches!(jit_module(&m), Err(Error::InvalidData))); + + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_data_relocs( + "short", + vec![0; 4], + vec![DataReloc { + offset: 0, + symbol: "f".to_string(), + }], + ); + assert!(matches!(jit_module(&m), Err(Error::InvalidRelocation))); + + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_bss("odd", 8); + m.globals[0].align = 3; + assert!(matches!(jit_module(&m), Err(Error::InvalidData))); + } + + #[test] + fn explicit_alignment_is_honored() { + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_writable("pad", vec![1]); + m.add_writable("aligned", vec![0; 8]); + m.globals[1].align = 64; + let jit = match jit_module(&m) { + Ok(jit) => jit, + Err(e) => panic!("jit_module failed: {e}"), + }; + assert_eq!(jit.data_addr("aligned").unwrap() as usize % 64, 0); + } +} diff --git a/src/target/src/regalloc/addrfold.rs b/src/target/src/regalloc/addrfold.rs index f0bd582..fafd96f 100644 --- a/src/target/src/regalloc/addrfold.rs +++ b/src/target/src/regalloc/addrfold.rs @@ -211,10 +211,7 @@ mod tests { use volt_ir::types::{IntDesc, TypeKind}; fn i32_ty(func: &mut Function) -> volt_ir::types::Type { - func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })) + func.types.intern(TypeKind::Int(IntDesc { bits: 32 })) } fn fold_in_range(func: &F, mem_inst: Inst) -> Option { @@ -251,7 +248,7 @@ mod tests { i32t, Opcode::Load(volt_ir::function::Load { ptr: p, - volatile: false, + mem: volt_ir::function::MemFlags::new(), }), ); func.set_terminator(b, Terminator::Ret(Ret::one(load))); @@ -280,7 +277,7 @@ mod tests { i32t, Opcode::Load(volt_ir::function::Load { ptr: base, - volatile: false, + mem: volt_ir::function::MemFlags::new(), }), ); func.set_terminator(b, Terminator::Ret(Ret::one(load))); @@ -312,7 +309,7 @@ mod tests { i32t, Opcode::Load(volt_ir::function::Load { ptr: p, - volatile: false, + mem: volt_ir::function::MemFlags::new(), }), ); func.set_terminator(b, Terminator::Ret(Ret::one(load))); @@ -344,7 +341,7 @@ mod tests { i32t, Opcode::Load(volt_ir::function::Load { ptr: p, - volatile: false, + mem: volt_ir::function::MemFlags::new(), }), ); let _ = load; diff --git a/src/target/src/regalloc/ir.rs b/src/target/src/regalloc/ir.rs index b9c9c8d..dd1a81d 100644 --- a/src/target/src/regalloc/ir.rs +++ b/src/target/src/regalloc/ir.rs @@ -143,6 +143,11 @@ pub fn visit_inst_operands(func: &F, inst: Inst, f: &mut impl FnMut(Val f(rd, false); } } + Opcode::Intrinsic(n) => { + for a in func.value_list(n.args) { + f(*a, false); + } + } Opcode::If(cf) => { f(cf.cond, false); for a in func.block_args(cf.then) { diff --git a/src/target/src/regalloc/wimmer.rs b/src/target/src/regalloc/wimmer.rs index 519f955..54662b7 100644 --- a/src/target/src/regalloc/wimmer.rs +++ b/src/target/src/regalloc/wimmer.rs @@ -3697,10 +3697,7 @@ mod tests { } fn i32_ty(func: &mut Function) -> volt_ir::types::Type { - func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })) + func.types.intern(TypeKind::Int(IntDesc { bits: 32 })) } #[test] diff --git a/src/target/src/x86_64/cases.rs b/src/target/src/x86_64/cases.rs new file mode 100644 index 0000000..dbf78cd --- /dev/null +++ b/src/target/src/x86_64/cases.rs @@ -0,0 +1,676 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Execution cases for the operations whose result depends on signedness. Each [`Case`] is one +//! function with the arguments to call it with and the value a correct execution returns; the +//! expected values come from the plain Rust reference arithmetic below, written independently +//! of the instruction selector. `tests.rs` executes the cases natively; `codegen_tests.rs` +//! compiles them on every host and checks the machine code decodes. +//! +//! Integer registers hold values narrower than 64 bits zero-extended, so a correct function +//! also returns them zero-extended, and ignores whatever the caller left above the declared +//! width of an argument. The cases pin both: arguments carry dirty upper bits on purpose and +//! results are compared exactly. +use alloc::{format, string::String, vec, vec::Vec}; +use volt_ir::function::*; + +/// An argument or return value. +#[derive(Clone, Copy, Debug)] +pub enum Val { + Int(u64), + F32(f32), + F64(f64), +} + +impl Val { + /// The raw register contents (an `f32` in the low 32 bits). + pub fn bits(self) -> u64 { + match self { + Val::Int(v) => v, + Val::F32(v) => v.to_bits() as u64, + Val::F64(v) => v.to_bits(), + } + } +} + +pub struct Case { + pub name: String, + pub function: Function, + /// `(arguments, expected result)` pairs. + pub runs: Vec<(Vec, Val)>, +} + +pub fn mask(bits: u32) -> u64 { + if bits >= 64 { + u64::MAX + } else { + (1 << bits) - 1 + } +} + +/// `v` read as a two's complement integer of `bits` bits. +pub fn sext(v: u64, bits: u32) -> i64 { + let s = 64 - bits; + ((v << s) as i64) >> s +} + +/// `a` with the bits above `bits` filled with garbage when `dirty` is odd. +fn dirty(a: u64, bits: u32, k: usize) -> u64 { + if k & 1 == 1 { a | !mask(bits) } else { a } +} + +fn build( + params: &[&str], + ret: &str, + body: impl FnOnce(&mut Function, Block, &[Value], volt_ir::types::Type) -> Value, +) -> Function { + let mut f = Function::new(); + let b = f.append_block(); + let args: Vec = params + .iter() + .map(|p| { + let t = f.types.parse_type(p).unwrap(); + f.append_block_param(b, t) + }) + .collect(); + let t = f.types.parse_type(ret).unwrap(); + let v = body(&mut f, b, &args, t); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + f +} + +pub fn arith(kind: &str, op: BinOp) -> Function { + build(&[kind, kind], kind, |f, b, a, t| { + f.append_inst( + b, + t, + Opcode::Arith(Arith { + op, + lhs: a[0], + rhs: a[1], + }), + ) + }) +} + +pub fn arith_imm(kind: &str, op: BinOp, imm: i64) -> Function { + build(&[kind], kind, |f, b, a, t| { + f.append_inst(b, t, Opcode::ArithImm(ArithImm { op, lhs: a[0], imm })) + }) +} + +pub fn compare(kind: &str, op: CmpOp) -> Function { + build(&[kind, kind], "bool", |f, b, a, t| { + f.append_inst( + b, + t, + Opcode::Icmp(Compare { + op, + lhs: a[0], + rhs: a[1], + }), + ) + }) +} + +pub fn convert(from: &str, to: &str, kind: ConvertKind) -> Function { + build(&[from], to, |f, b, a, t| { + f.append_inst(b, t, Opcode::Convert(Convert { value: a[0], kind })) + }) +} + +/// The value of `op` on two `bits`-bit integers, or `None` where the hardware traps (a zero +/// divisor, or the most negative value divided by -1 at the 32 and 64 bit widths). +pub fn arith_reference(op: BinOp, bits: u32, a: u64, b: u64) -> Option { + let (a, b) = (a & mask(bits), b & mask(bits)); + let (sa, sb) = (sext(a, bits), sext(b, bits)); + let trapping_signed = (bits == 32 || bits == 64) && sa == i64::MIN >> (64 - bits) && sb == -1; + let r = match op { + BinOp::Add => a.wrapping_add(b), + BinOp::Sub => a.wrapping_sub(b), + BinOp::Mul => a.wrapping_mul(b), + BinOp::BitAnd => a & b, + BinOp::BitOr => a | b, + BinOp::BitXor => a ^ b, + BinOp::Shl => a << b, + BinOp::Shr => a >> b, + BinOp::Sar => (sa >> b) as u64, + BinOp::UDiv if b != 0 => a / b, + BinOp::URem if b != 0 => a % b, + BinOp::SDiv if b != 0 && !trapping_signed => sa.wrapping_div(sb) as u64, + BinOp::SRem if b != 0 && !trapping_signed => sa.wrapping_rem(sb) as u64, + BinOp::UMulh => ((a as u128 * b as u128) >> bits) as u64, + BinOp::SMulh => ((sa as i128 * sb as i128) >> bits) as u64, + _ => return None, + }; + Some(r & mask(bits)) +} + +pub const WIDTHS: [u32; 6] = [8, 16, 32, 33, 48, 64]; + +pub fn samples(bits: u32) -> Vec { + let m = mask(bits); + let mut v = vec![ + 0, + 1, + 2, + 3, + 5, + 7, + 100, + 200, + m, + m - 1, + m >> 1, + (m >> 1) + 1, + ((m >> 1) + 1) | 1, + 0x5555_5555_5555_5555 & m, + 0xaaaa_aaaa_aaaa_aaaa & m, + 0x0123_4567_89ab_cdef & m, + ]; + for x in &mut v { + *x &= m; + } + v.sort_unstable(); + v.dedup(); + v +} + +fn is_shift(op: BinOp) -> bool { + matches!(op, BinOp::Shl | BinOp::Shr | BinOp::Sar) +} + +/// Constants: the payload is any `i64`, and the register must hold its low `bits` +/// zero-extended, whatever the sign of the payload. +fn iconst_cases(out: &mut Vec) { + for bits in WIDTHS { + let kind = format!("i{bits}"); + for n in [ + -1i64, + -2, + 1, + -100, + 127, + -128, + 255, + 0x1234_5678_9abc_def0, + i64::MIN, + ] { + out.push(Case { + name: format!("{kind} Iconst {n}"), + function: build(&[], &kind, |f, b, _, t| { + f.append_inst(b, t, Opcode::Iconst(n)) + }), + runs: vec![(vec![], Val::Int(n as u64 & mask(bits)))], + }); + } + } +} + +fn arith_cases(out: &mut Vec) { + for bits in WIDTHS { + let kind = format!("i{bits}"); + for op in [ + BinOp::Add, + BinOp::Sub, + BinOp::Mul, + BinOp::BitAnd, + BinOp::BitOr, + BinOp::BitXor, + BinOp::Shl, + BinOp::Shr, + BinOp::Sar, + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::UMulh, + BinOp::SMulh, + ] { + let mut runs = Vec::new(); + let mut k = 0; + for &a in &samples(bits) { + for &b in &samples(bits) { + // A shift amount is only meaningful below the width. + let b = if is_shift(op) { b % bits as u64 } else { b }; + if let Some(want) = arith_reference(op, bits, a, b) { + k += 1; + runs.push(( + vec![Val::Int(dirty(a, bits, k)), Val::Int(dirty(b, bits, k / 2))], + Val::Int(want), + )); + } + } + } + out.push(Case { + name: format!("{kind} {op:?}"), + function: arith(&kind, op), + runs, + }); + } + } +} + +/// `op` against a constant: the immediate is the value an `Iconst` of the operand type would +/// hold, so a negative immediate on a narrow type means its unsigned reading. +fn immediate_cases(out: &mut Vec) { + for (bits, op, imm) in [ + (8, BinOp::Add, -1), + (8, BinOp::UDiv, -56), + (8, BinOp::URem, -56), + (8, BinOp::SDiv, -3), + (8, BinOp::SRem, -3), + (8, BinOp::Shl, 3), + (8, BinOp::Shr, 3), + (8, BinOp::Sar, 3), + (16, BinOp::UDiv, -1000), + (16, BinOp::SRem, -1000), + (16, BinOp::Sar, 15), + (32, BinOp::UDiv, -3), + (32, BinOp::SDiv, -3), + (32, BinOp::SRem, -3), + (32, BinOp::Sar, 31), + (32, BinOp::Shr, 31), + (48, BinOp::UDiv, -3), + (48, BinOp::SDiv, -3), + (48, BinOp::Sar, 47), + (64, BinOp::SDiv, -3), + (64, BinOp::UDiv, -3), + (64, BinOp::Sar, 63), + (64, BinOp::Shr, 63), + ] { + let kind = format!("i{bits}"); + let mut runs = Vec::new(); + for (k, &a) in samples(bits).iter().enumerate() { + if let Some(want) = arith_reference(op, bits, a, imm as u64) { + runs.push((vec![Val::Int(dirty(a, bits, k))], Val::Int(want))); + } + } + out.push(Case { + name: format!("{kind} {op:?} imm {imm}"), + function: arith_imm(&kind, op, imm), + runs, + }); + } +} + +pub fn compare_reference(op: CmpOp, bits: u32, a: u64, b: u64) -> bool { + let (a, b) = (a & mask(bits), b & mask(bits)); + let (sa, sb) = (sext(a, bits), sext(b, bits)); + match op { + CmpOp::Eq => a == b, + CmpOp::Ne => a != b, + CmpOp::Slt => sa < sb, + CmpOp::Sle => sa <= sb, + CmpOp::Sgt => sa > sb, + CmpOp::Sge => sa >= sb, + CmpOp::Ult => a < b, + CmpOp::Ule => a <= b, + CmpOp::Ugt => a > b, + CmpOp::Uge => a >= b, + CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge => unreachable!("float compare"), + } +} + +fn compare_cases(out: &mut Vec) { + let signed = [CmpOp::Slt, CmpOp::Sle, CmpOp::Sgt, CmpOp::Sge]; + let unsigned = [CmpOp::Ult, CmpOp::Ule, CmpOp::Ugt, CmpOp::Uge]; + let equality = [CmpOp::Eq, CmpOp::Ne]; + // `ptr` is an unsigned 64-bit quantity: it has no signed order. + for kind in WIDTHS + .map(|b| format!("i{b}")) + .into_iter() + .chain(["ptr".into()]) + { + let bits = kind[1..].parse().unwrap_or(64); + let ops: Vec = if kind == "ptr" { + equality.iter().chain(&unsigned).copied().collect() + } else { + equality + .iter() + .chain(&signed) + .chain(&unsigned) + .copied() + .collect() + }; + for op in ops { + let mut runs = Vec::new(); + let mut k = 0; + for &a in &samples(bits) { + for &b in &samples(bits) { + k += 1; + runs.push(( + vec![Val::Int(dirty(a, bits, k)), Val::Int(dirty(b, bits, k / 3))], + Val::Int(compare_reference(op, bits, a, b) as u64), + )); + } + } + out.push(Case { + name: format!("{kind} {op:?}"), + function: compare(&kind, op), + runs, + }); + } + } +} + +fn integer_convert_cases(out: &mut Vec) { + // (from, to) widths of the integer width changes. + let trunc = [ + (64, 32), + (64, 8), + (64, 16), + (64, 48), + (48, 8), + (33, 8), + (32, 16), + (32, 8), + (16, 8), + ]; + let ext = [ + (8, 16), + (8, 32), + (8, 48), + (8, 64), + (16, 32), + (16, 64), + (32, 33), + (32, 64), + (33, 64), + (48, 64), + ]; + for (kind, pairs) in [ + (ConvertKind::Trunc, &trunc[..]), + (ConvertKind::Zext, &ext[..]), + (ConvertKind::Sext, &ext[..]), + ] { + for &(from, to) in pairs { + let mut runs = Vec::new(); + for (k, &a) in samples(from).iter().enumerate() { + let want = match kind { + ConvertKind::Trunc => a & mask(to), + ConvertKind::Zext => a, + _ => sext(a, from) as u64 & mask(to), + }; + runs.push((vec![Val::Int(dirty(a, from, k))], Val::Int(want))); + } + out.push(Case { + name: format!("i{from} {kind:?} i{to}"), + function: convert(&format!("i{from}"), &format!("i{to}"), kind), + runs, + }); + } + } + for to in [32, 64] { + out.push(Case { + name: format!("bool Zext i{to}"), + function: convert("bool", &format!("i{to}"), ConvertKind::Zext), + runs: vec![ + (vec![Val::Int(0)], Val::Int(0)), + (vec![Val::Int(1)], Val::Int(1)), + // Only the declared bit of the argument counts. + (vec![Val::Int(0xffff_ff00)], Val::Int(0)), + (vec![Val::Int(0xffff_ff01)], Val::Int(1)), + ], + }); + } +} + +fn float_convert_cases(out: &mut Vec) { + for bits in [8, 16, 32, 48, 64] { + for (fkind, double) in [("f64", true), ("f32", false)] { + for kind in [ConvertKind::SiToFp, ConvertKind::UiToFp] { + let mut runs = Vec::new(); + for (k, &a) in samples(bits).iter().enumerate() { + let want = match (kind == ConvertKind::SiToFp, double) { + (true, true) => Val::F64(sext(a, bits) as f64), + (true, false) => Val::F32(sext(a, bits) as f32), + (false, true) => Val::F64(a as f64), + (false, false) => Val::F32(a as f32), + }; + runs.push((vec![Val::Int(dirty(a, bits, k))], want)); + } + out.push(Case { + name: format!("i{bits} {kind:?} {fkind}"), + function: convert(&format!("i{bits}"), fkind, kind), + runs, + }); + } + } + } + // In-range sources only: out-of-range results are the hardware's "integer indefinite". + let signed_sources = |bits: u32| -> Vec { + match bits { + 8 => vec![0.0, 1.5, -1.5, 100.9, -100.9, 127.0, -128.0], + 16 => vec![0.0, 1.5, -1.5, 30000.7, -30000.7, 32767.0, -32768.0], + 32 => vec![0.0, 1.5, -1.5, 2.0e9, -2.0e9, 2147483520.0, -2147483648.0], + _ => vec![ + 0.0, + 1.5, + -1.5, + 1.0e15, + -1.0e15, + 9.0e18, + -9.0e18, + -9223372036854775808.0, + ], + } + }; + let unsigned_sources = |bits: u32| -> Vec { + match bits { + 8 => vec![0.0, 1.9, 200.5, 255.0], + 16 => vec![0.0, 1.9, 40000.5, 65535.0], + 32 => vec![0.0, 0.5, 3.0e9, 4294967295.0], + _ => vec![ + 0.0, + 1.5, + 9.0e18, + 9223372036854775807.0, + 9223372036854775808.0, + 1.8e19, + 18446744073709549568.0, + ], + } + }; + for bits in [8, 16, 32, 64] { + for (fkind, double) in [("f64", true), ("f32", false)] { + for kind in [ConvertKind::FpToSi, ConvertKind::FpToUi] { + let sources = if kind == ConvertKind::FpToSi { + signed_sources(bits) + } else { + unsigned_sources(bits) + }; + let mut runs = Vec::new(); + for v in sources { + // Round the source to the source type first; the reference then sees + // exactly the value the function is given. + let (arg, truncated) = if double { + (Val::F64(v), v) + } else { + (Val::F32(v as f32), v as f32 as f64) + }; + let half = 2f64.powi(bits as i32 - 1); + let in_range = if kind == ConvertKind::FpToSi { + truncated >= -half && truncated < half + } else { + truncated > -1.0 && truncated < 2.0 * half + }; + if !in_range { + continue; + } + let want = if kind == ConvertKind::FpToSi { + truncated as i64 as u64 & mask(bits) + } else { + truncated as u64 & mask(bits) + }; + runs.push((vec![arg], Val::Int(want))); + } + out.push(Case { + name: format!("{fkind} {kind:?} i{bits}"), + function: convert(fkind, &format!("i{bits}"), kind), + runs, + }); + } + } + } + out.push(Case { + name: "f32 FpResize f64".into(), + function: convert("f32", "f64", ConvertKind::FpResize), + runs: [0.0f32, 1.5, -2.25, 1.0e10, 3.0e38, 1.0e-30] + .map(|v| (vec![Val::F32(v)], Val::F64(v as f64))) + .into(), + }); + out.push(Case { + name: "f64 FpResize f32".into(), + function: convert("f64", "f32", ConvertKind::FpResize), + runs: [0.0f64, 1.5, -2.25, 1.0e10, 3.0e38, 0.1] + .map(|v| (vec![Val::F64(v)], Val::F32(v as f32))) + .into(), + }); +} + +/// `<4 x i32>` lanes: `lhs = [a0, a1, a1, a0]`, `rhs = [b0, b1, b0, b1]`, and the function +/// returns lane 0 | lane 2 << 32 of `lhs rhs`. +fn vector_function(op: Result) -> Function { + build(&["i32", "i32", "i32", "i32"], "i64", |f, b, p, wide| { + let vector = f.types.parse_type("<4 x i32>").unwrap(); + let int = f.types.parse_type("i32").unwrap(); + let fields = f.intern_values(&[p[0], p[1], p[1], p[0]]); + let lhs = f.append_inst(b, vector, Opcode::StructNew(StructNew { fields })); + let fields = f.intern_values(&[p[2], p[3], p[2], p[3]]); + let rhs = f.append_inst(b, vector, Opcode::StructNew(StructNew { fields })); + let v = match op { + Ok(op) => f.append_inst(b, vector, Opcode::Arith(Arith { op, lhs, rhs })), + Err(op) => f.append_inst(b, vector, Opcode::Icmp(Compare { op, lhs, rhs })), + }; + let lane = |f: &mut Function, index| { + let x = f.append_inst( + b, + int, + Opcode::Extract(Extract { + aggregate: v, + index, + }), + ); + f.append_inst( + b, + wide, + Opcode::Convert(Convert { + value: x, + kind: ConvertKind::Zext, + }), + ) + }; + let low = lane(f, 0); + let high = lane(f, 2); + let high = f.append_inst( + b, + wide, + Opcode::ArithImm(ArithImm { + op: BinOp::Shl, + lhs: high, + imm: 32, + }), + ); + f.append_inst( + b, + wide, + Opcode::Arith(Arith { + op: BinOp::BitOr, + lhs: low, + rhs: high, + }), + ) + }) +} + +fn vector_cases(out: &mut Vec) { + let lanes: [u64; 6] = [0, 1, 7, 0x7fff_ffff, 0x8000_0000, 0xffff_ffff]; + let combos = || { + lanes.into_iter().flat_map(move |a0| { + lanes.into_iter().flat_map(move |a1| { + lanes + .into_iter() + .flat_map(move |b0| lanes.into_iter().map(move |b1| [a0, a1, b0, b1])) + }) + }) + }; + for op in [ + BinOp::Add, + BinOp::Mul, + BinOp::Shr, + BinOp::Sar, + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::UMulh, + BinOp::SMulh, + ] { + let mut runs = Vec::new(); + for [a0, a1, b0, b1] in combos() { + let (b0, b1) = if is_shift(op) { + (b0 % 32, b1 % 32) + } else { + (b0, b1) + }; + // Every lane executes, so a lane that would trap rules the whole input out. + let lanes = [(a0, b0), (a1, b1), (a1, b0), (a0, b1)] + .map(|(a, b)| arith_reference(op, 32, a, b)); + if let [Some(l0), Some(_), Some(l2), Some(_)] = lanes { + runs.push(( + [a0, a1, b0, b1].map(Val::Int).into(), + Val::Int(l0 | l2 << 32), + )); + } + } + out.push(Case { + name: format!("<4 x i32> {op:?}"), + function: vector_function(Ok(op)), + runs, + }); + } + for op in [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Slt, + CmpOp::Sle, + CmpOp::Sgt, + CmpOp::Sge, + CmpOp::Ult, + CmpOp::Ule, + CmpOp::Ugt, + CmpOp::Uge, + ] { + let lane = |a, b| { + if compare_reference(op, 32, a, b) { + 0xffff_ffffu64 + } else { + 0 + } + }; + let runs = combos() + .map(|[a0, a1, b0, b1]| { + ( + [a0, a1, b0, b1].map(Val::Int).into(), + Val::Int(lane(a0, b0) | lane(a1, b0) << 32), + ) + }) + .collect(); + out.push(Case { + name: format!("<4 x i32> {op:?}"), + function: vector_function(Err(op)), + runs, + }); + } +} + +pub fn all() -> Vec { + let mut out = Vec::new(); + arith_cases(&mut out); + iconst_cases(&mut out); + immediate_cases(&mut out); + compare_cases(&mut out); + integer_convert_cases(&mut out); + float_convert_cases(&mut out); + vector_cases(&mut out); + out +} diff --git a/src/target/src/x86_64/codegen_tests.rs b/src/target/src/x86_64/codegen_tests.rs new file mode 100644 index 0000000..80dc398 --- /dev/null +++ b/src/target/src/x86_64/codegen_tests.rs @@ -0,0 +1,556 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Byte-level checks that signedness is chosen by the operation: the selector emits the signed +//! or the unsigned instruction (and the sign extension a signed operation needs) according to +//! the operator, never according to the type. They only compile code, so unlike `tests.rs` +//! (which executes it) they run on every host; the instruction text comes from this crate's +//! own disassembler, whose encodings are separately checked against llvm-mc. +use super::{Error, cases, compile, disasm}; +use alloc::{format, string::String, vec::Vec}; +use volt_ir::function::*; + +/// The disassembled instructions of `f`, one string per instruction. +fn asm(f: &Function) -> Vec { + let code = compile(f).expect("compiles"); + disasm::format(&code) + .lines() + .map(|l| l.split_once(": ").expect("offset prefix").1.into()) + .collect() +} + +fn has(lines: &[String], text: &str) -> bool { + lines.iter().any(|l| l == text) +} + +#[track_caller] +fn expect(lines: &[String], present: &[&str], absent: &[&str]) { + for p in present { + assert!(has(lines, p), "missing `{p}` in:\n{}", lines.join("\n")); + } + for a in absent { + assert!(!has(lines, a), "unexpected `{a}` in:\n{}", lines.join("\n")); + } +} + +#[test] +fn every_case_compiles_to_decodable_code() { + for case in cases::all() { + let code = compile(&case.function).unwrap_or_else(|e| panic!("{}: {e}", case.name)); + let text = disasm::format(&code); + assert!(!text.contains(".byte"), "{}:\n{text}", case.name); + check_case_shape(&case); + } +} + +/// The arguments and expected results of a case match the function's signature, and an +/// expected integer result is already zero-extended from its declared width. +fn check_case_shape(case: &cases::Case) { + use cases::Val; + use volt_ir::types::{FloatKind, TypeKind}; + let f = &case.function; + let class = |v: Value| match f.types.type_kind(f.value_type(v)) { + TypeKind::Float(FloatKind::F32) => "f32", + TypeKind::Float(FloatKind::F64) => "f64", + _ => "int", + }; + let params = f.block_params(Block(0)); + let Some(Terminator::Ret(ret)) = f.terminator(Block(0)) else { + panic!("{}: no return", case.name); + }; + let result = ret.slice()[0]; + let bits = match f.types.type_kind(f.value_type(result)) { + TypeKind::Int(i) => i.bits as u32, + TypeKind::Bool => 1, + _ => 64, + }; + assert!(!case.runs.is_empty(), "{}", case.name); + for (args, want) in &case.runs { + assert_eq!(args.len(), params.len(), "{}", case.name); + for (a, &p) in args.iter().zip(params) { + let want_class = match a { + Val::Int(_) => "int", + Val::F32(_) => "f32", + Val::F64(_) => "f64", + }; + assert_eq!(want_class, class(p), "{}", case.name); + } + let result_class = match want { + Val::Int(_) => "int", + Val::F32(_) => "f32", + Val::F64(_) => "f64", + }; + assert_eq!(result_class, class(result), "{}", case.name); + if let Val::Int(_) = want { + assert_eq!( + want.bits() & !cases::mask(bits), + 0, + "{} {args:x?}", + case.name + ); + } + } +} + +#[test] +fn division_follows_the_operator() { + for (kind, wide) in [("i32", false), ("i64", true)] { + let r11 = if wide { "r11" } else { "r11d" }; + let unsigned = asm(&cases::arith(kind, BinOp::UDiv)); + expect( + &unsigned, + &["xor edx, edx", &format!("div {r11}")], + &["cdq", "cqo", &format!("idiv {r11}")], + ); + let signed = asm(&cases::arith(kind, BinOp::SDiv)); + expect( + &signed, + &[if wide { "cqo" } else { "cdq" }, &format!("idiv {r11}")], + &["xor edx, edx", &format!("div {r11}")], + ); + // The remainder is the same divide read from rdx. + let urem = asm(&cases::arith(kind, BinOp::URem)); + let srem = asm(&cases::arith(kind, BinOp::SRem)); + expect(&urem, &[&format!("div {r11}"), "mov r10, rdx"], &[]); + expect(&srem, &[&format!("idiv {r11}"), "mov r10, rdx"], &[]); + } +} + +#[test] +fn narrow_signed_operations_sign_extend_their_operands_and_unsigned_ones_do_not() { + let sign_extension = ["shl r10, 56", "sar r10, 56", "shl r11, 56", "sar r11, 56"]; + for op in [BinOp::SDiv, BinOp::SRem, BinOp::SMulh] { + let lines = asm(&cases::arith("i8", op)); + expect(&lines, &sign_extension, &[]); + } + for op in [BinOp::UDiv, BinOp::URem, BinOp::UMulh] { + let lines = asm(&cases::arith("i8", op)); + expect(&lines, &[], &["sar r10, 56", "sar r11, 56"]); + } + // An arithmetic shift extends only the value shifted; a logical one extends nothing. + let sar = asm(&cases::arith("i8", BinOp::Sar)); + expect( + &sar, + &["shl r10, 56", "sar r10, 56", "sar r10d, cl"], + &["sar r11, 56"], + ); + let shr = asm(&cases::arith("i8", BinOp::Shr)); + expect(&shr, &["shr r10d, cl"], &["sar r10, 56", "sar r10d, cl"]); + // Wrapping operations need neither. + for op in [BinOp::Add, BinOp::Mul, BinOp::BitAnd] { + let lines = asm(&cases::arith("i8", op)); + expect(&lines, &[], &["sar r10, 56"]); + } +} + +#[test] +fn shifts_follow_the_operator() { + for (kind, r10) in [("i32", "r10d"), ("i64", "r10")] { + expect( + &asm(&cases::arith(kind, BinOp::Shl)), + &[&format!("shl {r10}, cl")], + &[&format!("shr {r10}, cl"), &format!("sar {r10}, cl")], + ); + expect( + &asm(&cases::arith(kind, BinOp::Shr)), + &[&format!("shr {r10}, cl")], + &[&format!("shl {r10}, cl"), &format!("sar {r10}, cl")], + ); + expect( + &asm(&cases::arith(kind, BinOp::Sar)), + &[&format!("sar {r10}, cl")], + &[&format!("shl {r10}, cl"), &format!("shr {r10}, cl")], + ); + } +} + +#[test] +fn high_product_follows_the_operator() { + expect( + &asm(&cases::arith("i32", BinOp::UMulh)), + &["mul r11d"], + &["imul r11d"], + ); + expect( + &asm(&cases::arith("i32", BinOp::SMulh)), + &["imul r11d"], + &["mul r11d"], + ); + expect( + &asm(&cases::arith("i64", BinOp::UMulh)), + &["mul r11"], + &["imul r11"], + ); + expect( + &asm(&cases::arith("i64", BinOp::SMulh)), + &["imul r11"], + &["mul r11"], + ); + // Narrower than 32 bits: a 64-bit product shifted down logically or arithmetically. + expect( + &asm(&cases::arith("i16", BinOp::UMulh)), + &["imul r10, r11", "shr r10, 16"], + &["sar r10, 16"], + ); + expect( + &asm(&cases::arith("i16", BinOp::SMulh)), + &["imul r10, r11", "sar r10, 16"], + &["shr r10, 16"], + ); + // Between 32 and 64 bits the double-width product spans rdx:rax. + expect( + &asm(&cases::arith("i48", BinOp::UMulh)), + &["mul r11"], + &["imul r11"], + ); + expect( + &asm(&cases::arith("i48", BinOp::SMulh)), + &["imul r11"], + &["mul r11"], + ); +} + +#[test] +fn comparisons_pick_the_condition_code_by_operator() { + // (operator, condition suffix of the set instruction) + for (op, cc) in [ + (CmpOp::Eq, "e"), + (CmpOp::Ne, "ne"), + (CmpOp::Slt, "l"), + (CmpOp::Sle, "le"), + (CmpOp::Sgt, "g"), + (CmpOp::Sge, "ge"), + (CmpOp::Ult, "b"), + (CmpOp::Ule, "be"), + (CmpOp::Ugt, "a"), + (CmpOp::Uge, "ae"), + ] { + for (kind, cmp) in [("i32", "cmp r10d, r11d"), ("i64", "cmp r10, r11")] { + let lines = asm(&cases::compare(kind, op)); + expect(&lines, &[cmp, &format!("set{cc} r10b")], &[]); + } + } + // Pointers compare unsigned. + expect( + &asm(&cases::compare("ptr", CmpOp::Ult)), + &["cmp r10, r11", "setb r10b"], + &["setl r10b"], + ); +} + +#[test] +fn narrow_signed_comparisons_sign_extend_both_operands() { + for op in [CmpOp::Slt, CmpOp::Sle, CmpOp::Sgt, CmpOp::Sge] { + expect( + &asm(&cases::compare("i8", op)), + &["shl r10, 56", "sar r10, 56", "shl r11, 56", "sar r11, 56"], + &[], + ); + expect( + &asm(&cases::compare("i48", op)), + &["shl r10, 16", "sar r10, 16", "cmp r10, r11"], + &[], + ); + } + for op in [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Ult, + CmpOp::Ule, + CmpOp::Ugt, + CmpOp::Uge, + ] { + expect( + &asm(&cases::compare("i8", op)), + &[], + &["sar r10, 56", "sar r11, 56"], + ); + } +} + +#[test] +fn integer_width_changes_pick_the_extension_by_conversion() { + use ConvertKind::*; + // 32 -> 64: a sign extension is `movsxd`; a zero extension needs no instruction at all, + // because a 32-bit value is already held zero-extended. + let sext = asm(&cases::convert("i32", "i64", Sext)); + let zext = asm(&cases::convert("i32", "i64", Zext)); + expect(&sext, &["movsxd r10, r10d"], &[]); + expect(&zext, &[], &["movsxd r10, r10d"]); + assert!(zext.len() < sext.len()); + // Narrower sources shift the sign (or zero) in. + expect( + &asm(&cases::convert("i8", "i64", Sext)), + &["shl r10, 56", "sar r10, 56"], + &[], + ); + expect( + &asm(&cases::convert("i8", "i64", Zext)), + &[], + &["sar r10, 56", "shr r10, 56"], + ); + // A sign extension to a narrower-than-64 destination is zero-extended afterwards. + expect( + &asm(&cases::convert("i8", "i32", Sext)), + &["shl r10, 56", "sar r10, 56", "mov r10d, r10d"], + &[], + ); + // Truncation keeps the low bits, zero-extended. + expect( + &asm(&cases::convert("i64", "i32", Trunc)), + &["mov r10d, r10d"], + &[], + ); + expect( + &asm(&cases::convert("i64", "i8", Trunc)), + &["shl r10, 56", "shr r10, 56"], + &["sar r10, 56"], + ); + expect( + &asm(&cases::convert("bool", "i64", Zext)), + &[], + &["sar r10, 56", "shr r10, 56"], + ); +} + +#[test] +fn integer_to_float_conversions_pick_the_signedness_by_conversion() { + use ConvertKind::*; + // 64-bit: the unsigned conversion needs the halve-and-double path for a set top bit. + let signed = asm(&cases::convert("i64", "f64", SiToFp)); + let unsigned = asm(&cases::convert("i64", "f64", UiToFp)); + expect(&signed, &["cvtsi2sd xmm13, r10"], &["test r10, r10"]); + expect( + &unsigned, + &["cvtsi2sd xmm13, r10", "test r10, r10", "addsd xmm13, xmm13"], + &[], + ); + expect( + &asm(&cases::convert("i64", "f32", UiToFp)), + &["cvtsi2ss xmm13, r10", "test r10, r10", "addss xmm13, xmm13"], + &[], + ); + // 32-bit: the signed conversion sign-extends first, the unsigned one converts the + // zero-extended value as a (non-negative) 64-bit integer. + let signed = asm(&cases::convert("i32", "f64", SiToFp)); + let unsigned = asm(&cases::convert("i32", "f64", UiToFp)); + expect(&signed, &["movsxd r10, r10d", "cvtsi2sd xmm13, r10"], &[]); + expect( + &unsigned, + &["cvtsi2sd xmm13, r10"], + &["movsxd r10, r10d", "test r10, r10"], + ); + // Narrow: sign-extended by shifts for the signed conversion only. + expect( + &asm(&cases::convert("i8", "f64", SiToFp)), + &["shl r10, 56", "sar r10, 56"], + &[], + ); + expect( + &asm(&cases::convert("i8", "f64", UiToFp)), + &["cvtsi2sd xmm13, r10"], + &["sar r10, 56"], + ); +} + +#[test] +fn float_to_integer_conversions_pick_the_signedness_by_conversion() { + use ConvertKind::*; + // 64-bit unsigned results above 2^63 take the subtract-and-flip path. + let signed = asm(&cases::convert("f64", "i64", FpToSi)); + let unsigned = asm(&cases::convert("f64", "i64", FpToUi)); + expect( + &signed, + &["cvttsd2si r10, xmm13"], + &["ucomisd xmm13, xmm14"], + ); + expect( + &unsigned, + &[ + "cvttsd2si r10, xmm13", + "ucomisd xmm13, xmm14", + "subsd xmm13, xmm14", + ], + &[], + ); + expect( + &asm(&cases::convert("f32", "i64", FpToUi)), + &[ + "cvttss2si r10, xmm13", + "ucomiss xmm13, xmm14", + "subss xmm13, xmm14", + ], + &[], + ); + // A signed 32-bit result is a 32-bit truncating convert; an unsigned one is a 64-bit + // convert (so values past 2^31 survive) whose low 32 bits are kept. + expect( + &asm(&cases::convert("f64", "i32", FpToSi)), + &["cvttsd2si r10d, xmm13", "mov r10d, r10d"], + &["cvttsd2si r10, xmm13"], + ); + expect( + &asm(&cases::convert("f64", "i32", FpToUi)), + &["cvttsd2si r10, xmm13", "mov r10d, r10d"], + &["cvttsd2si r10d, xmm13"], + ); +} + +#[test] +fn float_resize_converts_between_widths() { + use ConvertKind::*; + expect( + &asm(&cases::convert("f32", "f64", FpResize)), + &["cvtss2sd xmm13, xmm13"], + &["cvtsd2ss xmm13, xmm13"], + ); + expect( + &asm(&cases::convert("f64", "f32", FpResize)), + &["cvtsd2ss xmm13, xmm13"], + &["cvtss2sd xmm13, xmm13"], + ); +} + +#[test] +fn loads_zero_extend_and_never_sign_extend() { + for (kind, load) in [ + ("i8", "movzx r10, byte ptr [r11]"), + ("i16", "movzx r10, word ptr [r11]"), + ("i32", "mov r10d, dword ptr [r11]"), + ("i64", "mov r10, qword ptr [r11]"), + ] { + let mut f = Function::new(); + let p = f.types.parse_type("ptr").unwrap(); + let t = f.types.parse_type(kind).unwrap(); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let v = f.append_load(b, t, ptr, MemFlags::new()); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let lines = asm(&f); + expect(&lines, &[load], &[]); + assert!( + lines.iter().all(|l| !l.starts_with("movsx")), + "{kind}:\n{}", + lines.join("\n") + ); + } +} + +#[test] +fn narrow_arguments_and_call_results_are_zero_extended() { + // An argument is reloaded from its home slot at its declared width, which discards + // whatever the caller left above it. + let lines = asm(&cases::arith("i8", BinOp::Add)); + let loads = lines + .iter() + .filter(|l| l.starts_with("movzx r10, byte ptr [rbp - ")) + .count(); + assert_eq!(loads, 2, "{lines:?}"); + // A call result is cleaned before use: the ABI leaves the upper bits undefined. + let mut f = Function::new(); + let byte = f.types.parse_type("i8").unwrap(); + let p = f.types.parse_type("ptr").unwrap(); + let b = f.append_block(); + let target = f.append_block_param(b, p); + let args = f.intern_values(&[]); + let r = f.append_inst( + b, + byte, + Opcode::CallIndirect(CallIndirect { + target, + args, + is_variadic: false, + num_fixed: 0, + ret_dest: None, + ret_regs: 0, + ret_pieces: [RetPiece::default(); 4], + sret: false, + }), + ); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + expect(&asm(&f), &["shl rax, 56", "shr rax, 56"], &[]); +} + +#[test] +fn immediates_are_truncated_to_the_operand_width() { + // `udiv i8 x, -56` divides by 200, not by 2^64 - 56. + let lines = asm(&cases::arith_imm("i8", BinOp::UDiv, -56)); + expect(&lines, &["mov r11, 200"], &[]); + let lines = asm(&cases::arith_imm("i32", BinOp::UDiv, -3)); + expect(&lines, &["mov r11, 4294967293"], &[]); + let lines = asm(&cases::arith_imm("i64", BinOp::UDiv, -3)); + expect(&lines, &["mov r11, 18446744073709551613"], &[]); +} + +fn single(kind: &str, op: impl FnOnce(Value, Value) -> Opcode, ret: &str) -> Function { + let mut f = Function::new(); + let t = f.types.parse_type(kind).unwrap(); + let r = f.types.parse_type(ret).unwrap(); + let b = f.append_block(); + let x = f.append_block_param(b, t); + let y = f.append_block_param(b, t); + let v = f.append_inst(b, r, op(x, y)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + f +} + +#[test] +fn operations_applied_to_the_wrong_class_fail_closed() { + use Opcode::{Arith as A, Icmp}; + let arith = |op| move |lhs, rhs| A(Arith { op, lhs, rhs }); + let cmp = |op| move |lhs, rhs| Icmp(Compare { op, lhs, rhs }); + // The floating-point division on integers, the integer divisions on floats. + for op in [BinOp::Div, BinOp::Rem] { + let f = single("i32", arith(op), "i32"); + assert_eq!(compile(&f), Err(Error::InvalidFunction), "{op:?}"); + } + for op in [ + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::Shr, + BinOp::Sar, + ] { + let f = single("f64", arith(op), "f64"); + assert!(matches!(compile(&f), Err(Error::Unsupported(_))), "{op:?}"); + } + // The float orders on integers, the integer orders on floats. + for op in [CmpOp::Lt, CmpOp::Le, CmpOp::Gt, CmpOp::Ge] { + let f = single("i32", cmp(op), "bool"); + assert_eq!(compile(&f), Err(Error::InvalidFunction), "{op:?}"); + } + for op in [CmpOp::Slt, CmpOp::Ult, CmpOp::Sge, CmpOp::Uge] { + let f = single("f64", cmp(op), "bool"); + assert_eq!(compile(&f), Err(Error::InvalidFunction), "{op:?}"); + } + // An integer conversion of a float, a float conversion of the wrong class. + use ConvertKind::*; + for (from, to, kind) in [ + ("f64", "i64", Zext), + ("i32", "i64", FpResize), + ("i32", "f64", FpToSi), + ("f64", "f32", SiToFp), + ] { + let f = cases::convert(from, to, kind); + assert!(compile(&f).is_err(), "{from} {kind:?} {to}"); + } +} + +#[test] +fn integer_dot_products_are_not_lowered() { + let mut f = Function::new(); + let vector = f.types.parse_type("<4 x i32>").unwrap(); + let int = f.types.parse_type("i32").unwrap(); + let b = f.append_block(); + let acc = f.append_block_param(b, int); + let x = f.append_block_param(b, vector); + let y = f.append_block_param(b, vector); + for signed in [false, true] { + let mut f = f.clone_func(); + let v = f.append_dot(b, acc, x, y, signed); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + assert!( + matches!(compile(&f), Err(Error::Unsupported(_))), + "signed={signed}" + ); + } +} diff --git a/src/target/src/x86_64/disasm.rs b/src/target/src/x86_64/disasm.rs index b4d0903..fad8979 100644 --- a/src/target/src/x86_64/disasm.rs +++ b/src/target/src/x86_64/disasm.rs @@ -401,11 +401,15 @@ impl D<'_> { if op == 0xa8 { "al" } else { self.rv(0) }, self.imm(if op == 0xa8 { 1 } else { 4 })? ), - _ if (op <= 0x3b && op & 7 <= 3) || matches!(op, 0x84 | 0x85 | 0x88 | 0x89 | 0x8a) => { + _ if (op <= 0x3b && op & 7 <= 3) + || matches!(op, 0x84 | 0x85 | 0x86 | 0x87 | 0x88 | 0x89 | 0x8a) => + { let o = self.modrm()?; let byte = op & 1 == 0; let name = if op >= 0x88 { "mov" + } else if op >= 0x86 { + "xchg" } else if op >= 0x84 { "test" } else { @@ -425,7 +429,7 @@ impl D<'_> { self.rv(o.rm) }, ); - if op & 2 != 0 { + if op & 2 != 0 && !matches!(op, 0x86 | 0x87) { format!("{name} {reg}, {rm}") } else { format!("{name} {rm}, {reg}") @@ -815,6 +819,27 @@ mod tests { ); } #[test] + fn xchg_roundtrip_in_every_width() { + // Byte sequences checked against llvm-mc. + let mut e = Encoder::default(); + e.xchg(R11, 8, R10, 64); + assert_eq!(e.code, [0x4d, 0x87, 0x93, 8, 0, 0, 0]); + e.xchg(R11, 8, R10, 32); + e.xchg(R11, 8, R10, 16); + e.xchg(R11, 8, R10, 8); + e.xchg(RCX, 0, RAX, 8); + e.xchg(RCX, 0, RSI, 8); + assert_eq!( + format(&e.code), + "0000: xchg qword ptr [r11 + 8], r10\n\ + 0007: xchg dword ptr [r11 + 8], r10d\n\ + 000e: xchg word ptr [r11 + 8], r10w\n\ + 0016: xchg byte ptr [r11 + 8], r10b\n\ + 001d: xchg byte ptr [rcx], al\n\ + 0024: xchg byte ptr [rcx], sil\n" + ); + } + #[test] fn xmm_and_memory_roundtrip() { let mut e = Encoder::default(); e.xmm_mem(13, RBP, -32, 8, false); diff --git a/src/target/src/x86_64/encode.rs b/src/target/src/x86_64/encode.rs index f6983f3..4e22e6d 100644 --- a/src/target/src/x86_64/encode.rs +++ b/src/target/src/x86_64/encode.rs @@ -70,28 +70,13 @@ impl Encoder { self.code.push(0xb8 | (dst & 7)); self.bytes(&imm.to_le_bytes()); } - pub fn load(&mut self, dst: Reg, base: Reg, off: i32, bits: u16, signed: bool) { - match (bits, signed) { - (1..=8, s) => self.mem( - None, - &[0x0f, if s { 0xbe } else { 0xb6 }], - dst, - base, - off, - true, - false, - ), - (9..=16, s) => self.mem( - None, - &[0x0f, if s { 0xbf } else { 0xb7 }], - dst, - base, - off, - true, - false, - ), - (17..=32, true) => self.mem(None, &[0x63], dst, base, off, true, false), - (17..=32, false) => self.mem(None, &[0x8b], dst, base, off, false, false), + /// Loads `bits` of memory into `dst`, zero-extending to 64 bits. Sign extension is an + /// explicit operation of the instruction selector, never part of a load. + pub fn load(&mut self, dst: Reg, base: Reg, off: i32, bits: u16) { + match bits { + 1..=8 => self.mem(None, &[0x0f, 0xb6], dst, base, off, true, false), + 9..=16 => self.mem(None, &[0x0f, 0xb7], dst, base, off, true, false), + 17..=32 => self.mem(None, &[0x8b], dst, base, off, false, false), _ => self.mem(None, &[0x8b], dst, base, off, true, false), } } @@ -110,6 +95,23 @@ impl Encoder { bits <= 8, ); } + /// `xchg [base + off], src` of `bits` (8, 16, 32 or 64): swaps `src` with memory. With a + /// memory operand the exchange carries an implicit `lock`, so it is a full barrier. + pub fn xchg(&mut self, base: Reg, off: i32, src: Reg, bits: u16) { + self.mem( + if bits > 8 && bits <= 16 { + Some(0x66) + } else { + None + }, + &[if bits <= 8 { 0x86 } else { 0x87 }], + src, + base, + off, + bits > 32, + bits <= 8, + ); + } pub fn push(&mut self, reg: Reg) { self.rex(false, 0, reg, false); self.code.push(0x50 | (reg & 7)); diff --git a/src/target/src/x86_64/isel.rs b/src/target/src/x86_64/isel.rs index 575ea9f..5545fa4 100644 --- a/src/target/src/x86_64/isel.rs +++ b/src/target/src/x86_64/isel.rs @@ -10,6 +10,7 @@ use crate::regalloc::wimmer::{ UseKind, }; use alloc::{vec, vec::Vec}; +use volt_ir::attribute::Endianness; use volt_ir::function::*; use volt_ir::types::{FloatKind, Type, TypeKind}; @@ -88,7 +89,7 @@ fn shape(f: &Function, t: Type) -> Result<(usize, bool), Error> { && matches!( f.types.type_kind(v.elem), TypeKind::Float(FloatKind::F32) - | TypeKind::Int(volt_ir::types::IntDesc { bits: 32, .. }) + | TypeKind::Int(volt_ir::types::IntDesc { bits: 32 }) ) => { ((v.len * 4) as usize, true) @@ -114,6 +115,31 @@ fn size(f: &Function, t: Type) -> Result { _ => Ok(shape(f, t)?.0), } } +/// `n` truncated to `bits` and zero-extended: the canonical register form of an integer. +fn truncated(n: i64, bits: u16) -> u64 { + if bits < 64 { + n as u64 & ((1u64 << bits) - 1) + } else { + n as u64 + } +} +/// Condition code of an integer compare: equality, signed order (`l`, `le`, `g`, `ge`) or +/// unsigned order (`b`, `be`, `a`, `ae`). The floating-point orders have no integer meaning. +fn int_condition(op: CmpOp) -> Result { + Ok(match op { + CmpOp::Eq => 4, + CmpOp::Ne => 5, + CmpOp::Slt => 12, + CmpOp::Sle => 14, + CmpOp::Sgt => 15, + CmpOp::Sge => 13, + CmpOp::Ult => 2, + CmpOp::Ule => 6, + CmpOp::Ugt => 7, + CmpOp::Uge => 3, + CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge => return Err(Error::InvalidFunction), + }) +} struct Ctx<'a> { f: &'a Function, e: Encoder, @@ -151,9 +177,6 @@ impl Ctx<'_> { _ => 64, } } - fn signed(&self, v: Value) -> bool { - matches!(self.ty(v),TypeKind::Int(i) if i.signed) - } fn wide(&self, v: Value) -> bool { self.bytes(v) == 32 } @@ -181,7 +204,7 @@ impl Ctx<'_> { Location::Reg(s) => self.e.mov(r, s as u8), Location::Slot(s) => { let d = self.slot(0, s); - self.e.load(r, RBP, d, 64, false); + self.e.load(r, RBP, d, 64); } } } @@ -240,7 +263,7 @@ impl Ctx<'_> { self.e .xmm_mem(d as u8, RBP, off, if wide { 32 } else { 16 }, false) } else { - self.e.load(d as u8, RBP, off, 64, false) + self.e.load(d as u8, RBP, off, 64) } } (Location::Reg(s), Location::Slot(d)) => { @@ -261,7 +284,7 @@ impl Ctx<'_> { self.e .xmm_mem(15, RBP, de, if wide { 32 } else { 16 }, true) } else { - self.e.load(R11, RBP, so, 64, false); + self.e.load(R11, RBP, so, 64); self.e.store(RBP, de, R11, 64) } } @@ -282,15 +305,28 @@ impl Ctx<'_> { let at = self.e.branch(None); self.fixups.push((at, j.target)); } + /// Extends the low `bits` of `r` to 64 bits, sign-extending when `signed` and + /// zero-extending otherwise. Every integer narrower than 64 bits is held in its register + /// zero-extended; an operation that must read an operand as signed extends a scratch copy + /// of it with this first (`R10`/`R11` always hold such copies). fn normalize(&mut self, r: u8, bits: u16, signed: bool) { - if bits < 64 { - let shift = (64 - bits) as u8; - self.e.shift(4, r, Some(shift), true); - self.e - .shift(if signed { 7 } else { 5 }, r, Some(shift), true); + match (bits, signed) { + (64.., _) => {} + (32, false) => self.e.rr(None, &[0x89], r, r, false), + (32, true) => self.e.rr(None, &[0x63], r, r, true), + _ => { + let shift = (64 - bits) as u8; + self.e.shift(4, r, Some(shift), true); + self.e + .shift(if signed { 7 } else { 5 }, r, Some(shift), true); + } } } - fn int_binary_raw(&mut self, op: BinOp, wide: bool, signed: bool) { + /// `r10 = r10 r11` in 64-bit (`wide`) or 32-bit registers. A signed divide, remainder + /// or arithmetic shift reads its 32/64-bit operands as two's complement, which a + /// zero-extended 32-bit value already is; narrower operands must be sign-extended by the + /// caller (see `int_binary`). + fn int_binary_raw(&mut self, op: BinOp, wide: bool) -> Result<(), Error> { match op { BinOp::Add | BinOp::Sub | BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => self.e.rr( None, @@ -306,52 +342,66 @@ impl Ctx<'_> { wide, ), BinOp::Mul => self.e.rr(None, &[0x0f, 0xaf], R10, R11, wide), - BinOp::Shl | BinOp::Shr => { + BinOp::Shl | BinOp::Shr | BinOp::Sar => { self.e.mov(RCX, R11); self.e.shift( - if op == BinOp::Shl { - 4 - } else if signed { - 7 - } else { - 5 + match op { + BinOp::Shl => 4, + BinOp::Shr => 5, + _ => 7, }, R10, None, wide, ); } - BinOp::Div | BinOp::Rem => { + BinOp::UDiv | BinOp::URem => { self.e.mov(RAX, R10); - if signed { - if wide { - self.e.bytes(&[0x48, 0x99]); - } else { - self.e.bytes(&[0x99]); - } + self.e.rr(None, &[0x31], RDX, RDX, false); + self.e.rr(None, &[0xf7], 6, R11, wide); + self.e.mov(R10, if op == BinOp::UDiv { RAX } else { RDX }); + } + BinOp::SDiv | BinOp::SRem => { + self.e.mov(RAX, R10); + if wide { + self.e.bytes(&[0x48, 0x99]); } else { - self.e.rr(None, &[0x31], RDX, RDX, false); + self.e.bytes(&[0x99]); } - self.e - .rr(None, &[0xf7], if signed { 7 } else { 6 }, R11, wide); - self.e.mov(R10, if op == BinOp::Div { RAX } else { RDX }); + self.e.rr(None, &[0xf7], 7, R11, wide); + self.e.mov(R10, if op == BinOp::SDiv { RAX } else { RDX }); } - BinOp::Mulh => { + BinOp::UMulh | BinOp::SMulh => { self.e.mov(RAX, R10); - self.e - .rr(None, &[0xf7], if signed { 5 } else { 4 }, R11, wide); + self.e.rr( + None, + &[0xf7], + if op == BinOp::SMulh { 5 } else { 4 }, + R11, + wide, + ); self.e.mov(R10, RDX); } + // Floating-point only: `verify` rejects them on integers. + BinOp::Div | BinOp::Rem => return Err(Error::InvalidFunction), } + Ok(()) } - fn int_binary(&mut self, op: BinOp, lhs: Value, result: Value) -> Result<(), Error> { + fn int_binary(&mut self, op: BinOp, result: Value) -> Result<(), Error> { let bits = self.bits(result); - let signed = self.signed(lhs); - if bits != 32 && bits != 64 { - self.normalize(R10, bits, signed); - self.normalize(R11, bits, signed); + let narrow = bits != 32 && bits != 64; + if narrow { + match op { + BinOp::SDiv | BinOp::SRem | BinOp::SMulh => { + self.normalize(R10, bits, true); + self.normalize(R11, bits, true); + } + BinOp::Sar => self.normalize(R10, bits, true), + _ => {} + } } - if op == BinOp::Mulh && bits != 32 && bits != 64 { + if narrow && matches!(op, BinOp::UMulh | BinOp::SMulh) { + let signed = op == BinOp::SMulh; if bits < 32 { self.e.rr(None, &[0x0f, 0xaf], R10, R11, true); self.e @@ -366,38 +416,44 @@ impl Ctx<'_> { self.e.rr(None, &[0x09], RDX, R10, true); } } else { - self.int_binary_raw(op, bits > 32, signed); + self.int_binary_raw(op, bits > 32)?; } - if bits != 32 && bits != 64 { - self.normalize(R10, bits, self.signed(result)); + // Only these can leave bits above `bits` set in the result. + if narrow + && matches!( + op, + BinOp::Add + | BinOp::Sub + | BinOp::Mul + | BinOp::Shl + | BinOp::SDiv + | BinOp::SRem + | BinOp::Sar + | BinOp::SMulh + ) + { + self.normalize(R10, bits, false); } self.put(result, R10); Ok(()) } - fn integer_vector(&self, v: Value) -> Option { - match self.ty(v) { - TypeKind::Vector(v) => match self.f.types.type_kind(v.elem) { - TypeKind::Int(i) => Some(i.signed), - _ => None, - }, - _ => None, - } + fn integer_vector(&self, v: Value) -> bool { + matches!(self.ty(v), TypeKind::Vector(v) + if matches!(self.f.types.type_kind(v.elem), TypeKind::Int(_))) } - fn vector_integer_binary( - &mut self, - op: BinOp, - result: Value, - signed: bool, - ) -> Result<(), Error> { + fn scalar_float(&self, v: Value) -> bool { + matches!(self.ty(v), TypeKind::Float(k) if !matches!(k, FloatKind::F128)) + } + fn vector_integer_binary(&mut self, op: BinOp, result: Value) -> Result<(), Error> { let left = self.disp(self.temp); let right = self.disp(self.temp + 32); let out = self.disp(self.temp + 64); self.e.xmm_mem(13, RBP, left, self.bytes(result), true); self.e.xmm_mem(14, RBP, right, self.bytes(result), true); for lane in 0..self.bytes(result) / 4 { - self.e.load(R10, RBP, left + lane as i32 * 4, 32, signed); - self.e.load(R11, RBP, right + lane as i32 * 4, 32, signed); - self.int_binary_raw(op, false, signed); + self.e.load(R10, RBP, left + lane as i32 * 4, 32); + self.e.load(R11, RBP, right + lane as i32 * 4, 32); + self.int_binary_raw(op, false)?; self.e.store(RBP, out + lane as i32 * 4, R10, 32); } self.e.xmm_mem(13, RBP, out, self.bytes(result), false); @@ -405,8 +461,8 @@ impl Ctx<'_> { Ok(()) } fn float_binary(&mut self, op: BinOp, result: Value) -> Result<(), Error> { - if let Some(signed) = self.integer_vector(result) { - return self.vector_integer_binary(op, result, signed); + if self.integer_vector(result) { + return self.vector_integer_binary(op, result); } let opcode = match op { BinOp::Add => 0x58, @@ -439,10 +495,39 @@ impl Ctx<'_> { self.xput(result, 13); Ok(()) } + /// Rejects an ordered access with no instruction sequence: an ordering the access kind + /// cannot carry, or a type other than a 1/2/4/8-byte integer or pointer (legalize has + /// already turned atomic floats into integer accesses). + fn check_ordering(&self, mem: &MemFlags, v: Value, store: bool) -> Result<(), Error> { + let Some(ordering) = mem.ordering else { + return Ok(()); + }; + let scalar = match self.ty(v) { + TypeKind::Int(i) => matches!(i.bits, 8 | 16 | 32 | 64), + TypeKind::Ptr(_) => true, + _ => false, + }; + let allowed = if store { + matches!( + ordering, + AtomicOrdering::Relaxed | AtomicOrdering::Release | AtomicOrdering::SeqCst + ) + } else { + matches!( + ordering, + AtomicOrdering::Relaxed | AtomicOrdering::Acquire | AtomicOrdering::SeqCst + ) + }; + if scalar && allowed { + Ok(()) + } else { + Err(Error::Unsupported("unorderable atomic access")) + } + } fn memory_load(&mut self, result: Value, base: u8, off: i32) { if self.fp(result) { if self.half(result) { - self.e.load(R10, base, off, 16, false); + self.e.load(R10, base, off, 16); self.e.rr(Some(0x66), &[0x0f, 0x6e], 13, R10, false); self.e.vex_rr(2, 1, 0x13, 13, 0, 13, false); } else { @@ -450,8 +535,12 @@ impl Ctx<'_> { } self.xput(result, 13); } else { - self.e - .load(R10, base, off, self.bits(result), self.signed(result)); + let bits = self.bits(result); + self.e.load(R10, base, off, bits); + // A load reads a whole byte, word or doubleword: clear what lies above an odd width. + if !matches!(bits, 8 | 16 | 32 | 64) { + self.normalize(R10, bits, false); + } self.put(result, R10); } } @@ -471,7 +560,8 @@ impl Ctx<'_> { self.e.store(base, off, R10, self.bits(value)); } } - fn integer_vector_compare(&mut self, c: Compare, r: Value, signed: bool) { + fn integer_vector_compare(&mut self, c: Compare, r: Value) -> Result<(), Error> { + let cc = int_condition(c.op)?; let left = self.disp(self.temp); let right = self.disp(self.temp + 32); let out = self.disp(self.temp + 64); @@ -479,41 +569,9 @@ impl Ctx<'_> { self.xget(c.rhs, 14); self.e.xmm_mem(13, RBP, left, self.bytes(c.lhs), true); self.e.xmm_mem(14, RBP, right, self.bytes(c.rhs), true); - let cc = match c.op { - CmpOp::Eq => 4, - CmpOp::Ne => 5, - CmpOp::Lt => { - if signed { - 12 - } else { - 2 - } - } - CmpOp::Le => { - if signed { - 14 - } else { - 6 - } - } - CmpOp::Gt => { - if signed { - 15 - } else { - 7 - } - } - CmpOp::Ge => { - if signed { - 13 - } else { - 3 - } - } - }; for lane in 0..self.bytes(c.lhs) / 4 { - self.e.load(R10, RBP, left + lane as i32 * 4, 32, false); - self.e.load(R11, RBP, right + lane as i32 * 4, 32, false); + self.e.load(R10, RBP, left + lane as i32 * 4, 32); + self.e.load(R11, RBP, right + lane as i32 * 4, 32); self.e.rr(None, &[0x39], R11, R10, false); self.e.setcc(cc, R10); self.e.rr(None, &[0xf7], 3, R10, false); @@ -521,16 +579,23 @@ impl Ctx<'_> { } self.e.xmm_mem(13, RBP, out, self.bytes(r), false); self.xput(r, 13); + Ok(()) } fn comparison(&mut self, c: Compare, r: Value) -> Result<(), Error> { - if let Some(signed) = self.integer_vector(c.lhs) { - self.integer_vector_compare(c, r, signed); - return Ok(()); + if self.integer_vector(c.lhs) { + return self.integer_vector_compare(c, r); } if self.fp(c.lhs) { self.xget(c.lhs, 13); self.xget(c.rhs, 14); if matches!(self.ty(c.lhs), TypeKind::Vector(_)) { + let predicate = match c.op { + CmpOp::Eq => 0, + CmpOp::Ne => 4, + CmpOp::Lt | CmpOp::Gt => 1, + CmpOp::Le | CmpOp::Ge => 2, + _ => return Err(Error::InvalidFunction), + }; let swap = matches!(c.op, CmpOp::Gt | CmpOp::Ge); let (l, h) = if swap { (14, 13) } else { (13, 14) }; if self.wide(c.lhs) { @@ -539,12 +604,7 @@ impl Ctx<'_> { self.e.xmm_mov(15, l, false); self.e.rr(None, &[0x0f, 0xc2], 15, h, false); } - self.e.code.push(match c.op { - CmpOp::Eq => 0, - CmpOp::Ne => 4, - CmpOp::Lt | CmpOp::Gt => 1, - _ => 2, - }); + self.e.code.push(predicate); self.xput(r, 15); return Ok(()); } @@ -553,13 +613,6 @@ impl Ctx<'_> { "f128 compare requires softfp legalization", )); } - self.e.rr( - if self.double(c.lhs) { Some(0x66) } else { None }, - &[0x0f, 0x2e], - 13, - 14, - false, - ); let cc = match c.op { CmpOp::Eq => 4, CmpOp::Ne => 5, @@ -567,7 +620,15 @@ impl Ctx<'_> { CmpOp::Le => 6, CmpOp::Gt => 7, CmpOp::Ge => 3, + _ => return Err(Error::InvalidFunction), }; + self.e.rr( + if self.double(c.lhs) { Some(0x66) } else { None }, + &[0x0f, 0x2e], + 13, + 14, + false, + ); self.e.setcc(cc, R10); if matches!(c.op, CmpOp::Eq | CmpOp::Lt | CmpOp::Le) { self.e.setcc(11, R11); @@ -578,63 +639,47 @@ impl Ctx<'_> { } self.put(r, R10); } else { + let cc = int_condition(c.op)?; + let bits = self.bits(c.lhs); self.get(c.lhs, R10); self.get(c.rhs, R11); - self.e.rr(None, &[0x39], R11, R10, self.bits(c.lhs) > 32); - let signed = self.signed(c.lhs); - let cc = match c.op { - CmpOp::Eq => 4, - CmpOp::Ne => 5, - CmpOp::Lt => { - if signed { - 12 - } else { - 2 - } - } - CmpOp::Le => { - if signed { - 14 - } else { - 6 - } - } - CmpOp::Gt => { - if signed { - 15 - } else { - 7 - } - } - CmpOp::Ge => { - if signed { - 13 - } else { - 3 - } - } - }; + // A signed order reads the operands as two's complement of `bits`, which the + // zero-extended form only is at 32 and 64 bits. + if matches!(c.op, CmpOp::Slt | CmpOp::Sle | CmpOp::Sgt | CmpOp::Sge) + && bits != 32 + && bits != 64 + { + self.normalize(R10, bits, true); + self.normalize(R11, bits, true); + } + self.e.rr(None, &[0x39], R11, R10, bits > 32); self.e.setcc(cc, R10); self.put(r, R10); } Ok(()) } - fn convert(&mut self, s: Value, r: Value) -> Result<(), Error> { - match (self.fp(s), self.fp(r)) { - (false, false) => { + fn convert(&mut self, c: Convert, r: Value) -> Result<(), Error> { + let s = c.value; + let (from, to) = (self.bits(s), self.bits(r)); + match c.kind { + ConvertKind::Trunc | ConvertKind::Zext | ConvertKind::Sext => { + if self.fp(s) || self.fp(r) { + return Err(Error::InvalidFunction); + } self.get(s, R10); - if self.bits(r) > self.bits(s) { - self.normalize(R10, self.bits(s), self.signed(s)); + match c.kind { + // The source is already zero-extended: nothing to do. + ConvertKind::Zext => {} + ConvertKind::Trunc => self.normalize(R10, to, false), + _ => { + self.normalize(R10, from, true); + self.normalize(R10, to, false); + } } - self.normalize(R10, self.bits(r), self.signed(r)); self.put(r, R10); } - (true, true) => { - if matches!(self.ty(s), TypeKind::Vector(_)) - || matches!(self.ty(r), TypeKind::Vector(_)) - || self.bytes(s) == 16 - || self.bytes(r) == 16 - { + ConvertKind::FpResize => { + if !self.scalar_float(s) || !self.scalar_float(r) { return Err(Error::Unsupported("composite conversion")); } self.xget(s, 13); @@ -652,11 +697,18 @@ impl Ctx<'_> { } self.xput(r, 13); } - (false, true) => { + ConvertKind::SiToFp | ConvertKind::UiToFp => { + if self.fp(s) || !self.scalar_float(r) { + return Err(Error::Unsupported("composite conversion")); + } self.get(s, R10); - self.normalize(R10, self.bits(s), self.signed(s)); let p = Some(if self.double(r) { 0xf2 } else { 0xf3 }); - if !self.signed(s) && self.bits(s) == 64 { + if c.kind == ConvertKind::SiToFp { + self.normalize(R10, from, true); + self.e.rr(p, &[0x0f, 0x2a], 13, R10, true); + } else if from == 64 { + // A value with the top bit set does not fit a signed convert: halve it + // (keeping the low bit for rounding), convert, and double the result. self.e.rr(None, &[0x85], R10, R10, true); let normal = self.e.branch(Some(9)); self.e.mov(R11, R10); @@ -677,10 +729,14 @@ impl Ctx<'_> { } self.xput(r, 13); } - (true, false) => { + ConvertKind::FpToSi | ConvertKind::FpToUi => { + if !self.scalar_float(s) || self.fp(r) { + return Err(Error::Unsupported("composite conversion")); + } self.xget(s, 13); let prefix = Some(if self.double(s) { 0xf2 } else { 0xf3 }); - if !self.signed(r) && self.bits(r) == 64 { + let unsigned = c.kind == ConvertKind::FpToUi; + if unsigned && to == 64 { self.e.imm( R10, if self.double(s) { @@ -708,15 +764,10 @@ impl Ctx<'_> { self.e.rr(prefix, &[0x0f, 0x2c], R10, 13, true); self.e.patch(end, self.e.code.len())?; } else { - self.e.rr( - prefix, - &[0x0f, 0x2c], - R10, - 13, - self.bits(r) > 32 || !self.signed(r), - ); + self.e + .rr(prefix, &[0x0f, 0x2c], R10, 13, to > 32 || unsigned); } - self.normalize(R10, self.bits(r), false); + self.normalize(R10, to, false); self.put(r, R10); } } @@ -747,7 +798,7 @@ impl Ctx<'_> { if self.fp(v) { if xi < 8 { if self.half(v) { - self.e.load(R10, RBP, off, 16, false); + self.e.load(R10, RBP, off, 16); self.e.rr(Some(0x66), &[0x0f, 0x6e], xi, R10, false); } else { self.e.xmm_mem(xi, RBP, off, self.bytes(v), false); @@ -762,16 +813,11 @@ impl Ctx<'_> { stack += side; } } else if gi < 6 { - self.e.load( - [RDI, RSI, RDX, RCX, R8, R9][gi], - RBP, - off, - self.bits(v), - self.signed(v), - ); + self.e + .load([RDI, RSI, RDX, RCX, R8, R9][gi], RBP, off, self.bits(v)); gi += 1; } else { - self.e.load(R11, RBP, off, 64, false); + self.e.load(R11, RBP, off, 64); self.e.store(RSP, stack, R11, 64); stack += 8; } @@ -780,7 +826,7 @@ impl Ctx<'_> { self.e.imm(RAX, xi as u64); } if target.is_some() { - self.e.load(R10, RBP, self.disp(self.temp), 64, false); + self.e.load(R10, RBP, self.disp(self.temp), 64); self.e.rr(None, &[0xff], 2, R10, false); } else { self.e.code.push(0xe8); @@ -803,6 +849,8 @@ impl Ctx<'_> { } self.xput(r, 0); } else { + // The ABI leaves the bits above a narrow integer return undefined. + self.normalize(RAX, self.bits(r), false); self.put(r, RAX); } } @@ -884,18 +932,18 @@ impl Ctx<'_> { let limit = if fp { 176 } else { 48 }; let step = if fp { 16 } else { 8 }; self.get(list, R11); - self.e.load(R10, R11, field, 32, false); + self.e.load(R10, R11, field, 32); self.e.alu_imm(7, R10, limit, false); let overflow = self.e.branch(Some(3)); self.e.mov(RCX, R10); - self.e.load(RAX, R11, 16, 64, false); + self.e.load(RAX, R11, 16, 64); self.e.rr(None, &[0x01], R10, RAX, true); self.e.alu_imm(0, RCX, step, false); self.e.store(R11, field, RCX, 32); self.memory_load(result, RAX, 0); let end = self.e.branch(None); self.e.patch(overflow, self.e.code.len())?; - self.e.load(RAX, R11, 8, 64, false); + self.e.load(RAX, R11, 8, 64); self.e.mov(RCX, RAX); self.e.alu_imm(0, RCX, 8, true); self.e.store(R11, 8, RCX, 64); @@ -921,8 +969,22 @@ impl Ctx<'_> { let op = self.f.opcode(i); match op { Opcode::Store(s) => { + if s.mem.endian == Endianness::Big { + return Err(Error::Unsupported("big-endian store")); + } + self.check_ordering(&s.mem, s.value, true)?; self.get(s.ptr, R11); - self.memory_store(s.value, R11, self.fold.off_of(i) as i32); + let off = self.fold.off_of(i) as i32; + if s.mem.ordering == Some(AtomicOrdering::SeqCst) { + // x86-64 (TSO) already orders relaxed and release stores with plain `mov`; + // only a sequentially consistent store must also not reorder with a later + // load, which `xchg` (implicitly locked) guarantees. It leaves the old + // memory value in R10, a scratch register. + self.get(s.value, R10); + self.e.xchg(R11, off, R10, self.bits(s.value)); + } else { + self.memory_store(s.value, R11, off); + } } Opcode::Prefetch(p) => { self.get(p.ptr, R11); @@ -938,12 +1000,12 @@ impl Ctx<'_> { } Opcode::VaStart(v) => self.va_start(v.list), Opcode::VaEnd(_) => {} + Opcode::Intrinsic(_) => return Err(Error::Unsupported("intrinsic")), _ => { let r = result.ok_or(Error::Unsupported("statement"))?; match op { Opcode::Iconst(n) => { - self.e.imm(R10, n as u64); - self.normalize(R10, self.bits(r), self.signed(r)); + self.e.imm(R10, truncated(n, self.bits(r))); self.put(r, R10); } Opcode::Fconst(n) => { @@ -980,7 +1042,7 @@ impl Ctx<'_> { } else { self.get(a.lhs, R10); self.get(a.rhs, R11); - self.int_binary(a.op, a.lhs, r)?; + self.int_binary(a.op, r)?; } } Opcode::ArithImm(a) => { @@ -988,8 +1050,8 @@ impl Ctx<'_> { return Err(Error::Unsupported("floating immediate arithmetic")); } self.get(a.lhs, R10); - self.e.imm(R11, a.imm as u64); - self.int_binary(a.op, a.lhs, r)?; + self.e.imm(R11, truncated(a.imm, self.bits(r))); + self.int_binary(a.op, r)?; } Opcode::Icmp(c) => self.comparison(c, r)?, Opcode::Select(s) => { @@ -1018,7 +1080,7 @@ impl Ctx<'_> { self.e.patch(end, self.e.code.len())?; } } - Opcode::Convert(c) => self.convert(c.value, r)?, + Opcode::Convert(c) => self.convert(c, r)?, Opcode::Unary(u) => { if u.op == UnaryOp::Reinterpret { if self.fp(u.value) == self.fp(r) { @@ -1088,6 +1150,13 @@ impl Ctx<'_> { } } Opcode::Load(l) => { + if l.mem.endian == Endianness::Big { + return Err(Error::Unsupported("big-endian load")); + } + self.check_ordering(&l.mem, r, false)?; + // Every permitted load ordering is a plain `mov` on x86-64: TSO gives + // loads acquire semantics, and a sequentially consistent load needs + // no extra instruction because the store side is the locked one. self.get(l.ptr, R11); self.memory_load(r, R11, self.fold.off_of(i) as i32); } @@ -1158,15 +1227,18 @@ impl Ctx<'_> { } self.xput(r, 13); } else { - self.e.load(R10, RBP, off, 32, false); + self.e.load(R10, RBP, off, 32); for j in 1..self.bytes(rd.vector) / 4 { - self.e.load(R11, RBP, off + (j * 4) as i32, 32, false); - self.int_binary(rd.op, r, r)?; + self.e.load(R11, RBP, off + (j * 4) as i32, 32); + self.int_binary(rd.op, r)?; } self.put(r, R10); } } Opcode::Dot(d) => { + if self.integer_vector(d.a) { + return Err(Error::Unsupported("integer dot product")); + } let off = self.disp(self.temp); self.xget(d.a, 13); self.xget(d.b, 14); @@ -1209,7 +1281,7 @@ impl Ctx<'_> { return Err(Error::Unsupported("too many FP returns")); } if self.half(v) { - self.e.load(R10, RBP, off, 16, false); + self.e.load(R10, RBP, off, 16); self.e.rr(Some(0x66), &[0x0f, 0x6e], xi, R10, false); } else { self.e.xmm_mem(xi, RBP, off, self.bytes(v), false); @@ -1219,13 +1291,12 @@ impl Ctx<'_> { if gi >= 2 { return Err(Error::Unsupported("too many integer returns")); } - self.e - .load([RAX, RDX][gi], RBP, off, self.bits(v), self.signed(v)); + self.e.load([RAX, RDX][gi], RBP, off, self.bits(v)); gi += 1; } } if let Some(home) = self.sret_home { - self.e.load(RAX, RBP, self.disp(home), 64, false); + self.e.load(RAX, RBP, self.disp(home), 64); } if self.stack_alignment > 16 { self.e.mem( @@ -1492,7 +1563,7 @@ fn compile_lowered( c.e.store(RBP, off, [RDI, RSI, RDX, RCX, R8, R9][gi], 64); gi += 1; } else { - c.e.load(R10, RBP, stack, 64, false); + c.e.load(R10, RBP, stack, 64); c.e.store(RBP, off, R10, 64); stack += 8; } @@ -1542,3 +1613,131 @@ fn compile_lowered( relocs: c.relocs, }) } + +/// Byte-level checks of ordered accesses. They only compile code, so unlike `tests.rs` (which +/// executes it) they run on every host. +#[cfg(test)] +mod ordered_access_codegen { + use crate::x86_64::{Error, compile}; + use volt_ir::function::*; + + fn store_fn(kind: &str, ordering: Option) -> Function { + let mut f = Function::new(); + let p = f.types.parse_type("ptr").unwrap(); + let t = f.types.parse_type(kind).unwrap(); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let x = f.append_block_param(b, t); + let mem = MemFlags { + ordering, + ..MemFlags::new() + }; + f.append_store_mem(b, x, ptr, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + f + } + + fn load_fn(kind: &str, ordering: Option) -> Function { + let mut f = Function::new(); + let p = f.types.parse_type("ptr").unwrap(); + let t = f.types.parse_type(kind).unwrap(); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let mem = MemFlags { + ordering, + ..MemFlags::new() + }; + let v = f.append_load(b, t, ptr, mem); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + f + } + + fn contains(code: &[u8], pattern: &[u8]) -> bool { + code.windows(pattern.len()).any(|w| w == pattern) + } + + /// `[prefix] REX opcode modrm disp32=0` for `[r11]` and `r10`: the shape of both the plain + /// `mov [r11], r10` store and the `xchg [r11], r10` the encoder emits (R10 holds the value, + /// R11 the address). + fn r11_r10(prefix: &[u8], rex: u8, opcode: u8) -> Vec { + let mut bytes = prefix.to_vec(); + bytes.extend_from_slice(&[rex, opcode, 0x93, 0, 0, 0, 0]); + bytes + } + + #[test] + fn seq_cst_stores_are_xchg_and_weaker_stores_are_plain_mov() { + // (type, operand-size prefix, REX, xchg opcode, mov opcode) + for (kind, prefix, rex, xchg, mov) in [ + ("i8", &[][..], 0x45u8, 0x86u8, 0x88u8), + ("i16", &[0x66][..], 0x45, 0x87, 0x89), + ("i32", &[][..], 0x45, 0x87, 0x89), + ("i64", &[][..], 0x4d, 0x87, 0x89), + ] { + let xchg_bytes = r11_r10(prefix, rex, xchg); + let mov_bytes = r11_r10(prefix, rex, mov); + let plain = compile(&store_fn(kind, None)).unwrap(); + assert!(contains(&plain, &mov_bytes), "{kind} plain store"); + assert!(!contains(&plain, &xchg_bytes), "{kind} plain store"); + for ordering in [AtomicOrdering::Relaxed, AtomicOrdering::Release] { + let code = compile(&store_fn(kind, Some(ordering))).unwrap(); + assert_eq!(code, plain, "{kind} {ordering:?} must be a plain store"); + } + let seq_cst = compile(&store_fn(kind, Some(AtomicOrdering::SeqCst))).unwrap(); + assert!(contains(&seq_cst, &xchg_bytes), "{kind} seq_cst store"); + assert!(!contains(&seq_cst, &mov_bytes), "{kind} seq_cst store"); + assert_eq!(seq_cst.len(), plain.len(), "{kind}: xchg replaces the mov"); + } + } + + #[test] + fn ordered_loads_are_plain_movs() { + for kind in ["i8", "i16", "i32", "i64", "ptr"] { + let plain = compile(&load_fn(kind, None)).unwrap(); + for ordering in [ + AtomicOrdering::Relaxed, + AtomicOrdering::Acquire, + AtomicOrdering::SeqCst, + ] { + let code = compile(&load_fn(kind, Some(ordering))).unwrap(); + assert_eq!(code, plain, "{kind} {ordering:?} must be a plain load"); + } + } + } + + #[test] + fn atomic_float_accesses_use_the_integer_register_path() { + // The float is reinterpreted into an integer register first, so an atomic f64 store + // compiles to the same `xchg` an atomic u64 store does. + let mut f = Function::new(); + let p = f.types.parse_type("ptr").unwrap(); + let t = f.types.parse_type("f64").unwrap(); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let x = f.append_block_param(b, t); + let mem = MemFlags { + ordering: Some(AtomicOrdering::SeqCst), + ..MemFlags::new() + }; + f.append_store_mem(b, x, ptr, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let code = compile(&f).unwrap(); + assert!(contains(&code, &r11_r10(&[], 0x4d, 0x87))); + } + + #[test] + fn orderings_without_an_instruction_fail_closed() { + for ordering in [AtomicOrdering::Release, AtomicOrdering::AcqRel] { + assert!(matches!( + compile(&load_fn("i32", Some(ordering))), + Err(Error::Unsupported(_)) + )); + } + for ordering in [AtomicOrdering::Acquire, AtomicOrdering::AcqRel] { + assert!(matches!( + compile(&store_fn("i32", Some(ordering))), + Err(Error::Unsupported(_)) + )); + } + } +} diff --git a/src/target/src/x86_64/link.rs b/src/target/src/x86_64/link.rs index 9756280..7886f66 100644 --- a/src/target/src/x86_64/link.rs +++ b/src/target/src/x86_64/link.rs @@ -6,74 +6,7 @@ use super::{ isel::{self, RelocKind}, }; use alloc::{string::String, vec::Vec}; -use volt_ir::function::Function; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DataKind { - Rodata, - Data, - Bss, -} -#[derive(Clone, Debug)] -pub struct DataReloc { - pub off: usize, - pub symbol: String, -} -#[derive(Clone, Debug)] -pub struct Data { - pub name: String, - pub bytes: Vec, - pub kind: DataKind, - pub size: u64, - pub relocs: Vec, -} -#[derive(Clone, Debug, Default)] -pub struct Module { - pub functions: Vec<(String, Function)>, - pub data: Vec, -} -impl Module { - pub fn new() -> Self { - Self::default() - } - pub fn add_function(&mut self, name: &str, function: Function) { - self.functions.push((name.into(), function)); - } - pub fn add_data(&mut self, name: &str, bytes: Vec) { - self.add_data_relocs(name, bytes, Vec::new()); - } - pub fn add_data_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.into(), - bytes, - kind: DataKind::Rodata, - size, - relocs, - }); - } - pub fn add_writable(&mut self, name: &str, bytes: Vec) { - self.add_writable_relocs(name, bytes, Vec::new()); - } - pub fn add_writable_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.into(), - bytes, - kind: DataKind::Data, - size, - relocs, - }); - } - pub fn add_bss(&mut self, name: &str, size: u64) { - self.data.push(Data { - name: name.into(), - bytes: Vec::new(), - kind: DataKind::Bss, - size, - relocs: Vec::new(), - }); - } -} +use volt_ir::module::{Global, Module}; #[derive(Clone, Debug)] pub struct Symbol { pub name: String, @@ -91,12 +24,13 @@ pub struct Reloc { pub struct Linked { pub code: Vec, pub symbols: Vec, - pub data: Vec, + pub data: Vec, pub relocs: Vec, } pub fn compile_module(module: &Module) -> Result { let mut result = Linked::default(); - for (name, f) in &module.functions { + for def in &module.functions { + let (name, f) = (&def.name, &def.function); while result.code.len() % 16 != 0 { result.code.push(0x90); } @@ -134,18 +68,34 @@ pub fn compile_module(module: &Module) -> Result { pending.push(r); } result.relocs = pending; - result.data = module.data.clone(); + result.data = module.globals.clone(); Ok(result) } #[cfg(test)] mod tests { use super::*; - use volt_ir::function::{Call, Opcode, Ret, RetPiece, Terminator}; + use volt_ir::function::{Call, Function, Opcode, Ret, RetPiece, Terminator}; + use volt_ir::module::DataKind; + #[test] + fn carries_globals_into_the_linked_image() { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let v = f.append_inst(block, ty, Opcode::Iconst(1)); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + let mut module = Module::new(); + module.add_function("f", f); + module.add_data("ro", alloc::vec![1, 2, 3]); + module.add_bss("zero", 32); + let linked = compile_module(&module).unwrap(); + assert_eq!(linked.data, module.globals); + assert_eq!(linked.data[1].kind, DataKind::Bss); + } #[test] fn resolves_inter_function_call_displacement() { let mut caller = Function::new(); - let ty = caller.types.parse_type("u64").unwrap(); + let ty = caller.types.parse_type("i64").unwrap(); let block = caller.append_block(); let symbol = caller.intern_symbol("callee"); let args = caller.intern_values(&[]); @@ -165,7 +115,7 @@ mod tests { ); caller.set_terminator(block, Terminator::Ret(Ret::one(result))); let mut callee = Function::new(); - let ty = callee.types.parse_type("u64").unwrap(); + let ty = callee.types.parse_type("i64").unwrap(); let block = callee.append_block(); let result = callee.append_inst(block, ty, Opcode::Iconst(42)); callee.set_terminator(block, Terminator::Ret(Ret::one(result))); diff --git a/src/target/src/x86_64/mod.rs b/src/target/src/x86_64/mod.rs index da9d20d..c98c4e5 100644 --- a/src/target/src/x86_64/mod.rs +++ b/src/target/src/x86_64/mod.rs @@ -36,5 +36,9 @@ pub fn compile(function: &volt_ir::function::Function) -> Result Result, Error> { let mut obj = Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); let mut symbols = Vec::new(); let mut pending = Vec::new(); - for (name, f) in &module.functions { + for def in &module.functions { + let (name, f) = (&def.name, &def.function); let compiled = isel::compile_object(f)?; let section = obj.add_section( Vec::new(), @@ -58,7 +59,7 @@ pub fn write_module(module: &Module) -> Result, Error> { )); } } - for data in &module.data { + for data in &module.globals { let (class, kind) = match data.kind { DataKind::Rodata => ("rodata", SectionKind::ReadOnlyData), DataKind::Data => ("data", SectionKind::Data), @@ -69,10 +70,11 @@ pub fn write_module(module: &Module) -> Result, Error> { format!(".{class}.{}", data.name.trim_start_matches('.')).into_bytes(), kind, ); + let align = data.effective_align().max(8) as u64; if data.kind == DataKind::Bss { - obj.append_section_bss(sec, data.size, 8); + obj.append_section_bss(sec, data.size, align); } else { - obj.append_section_data(sec, &data.bytes, 8); + obj.append_section_data(sec, &data.bytes, align); } let id = obj.add_symbol(Symbol { name: data.name.as_bytes().to_vec(), @@ -90,13 +92,13 @@ pub fn write_module(module: &Module) -> Result, Error> { }); symbols.push((data.name.clone(), id)); for r in &data.relocs { - if r.off + if r.offset .checked_add(8) .is_none_or(|end| end > data.size as usize) { return Err(Error::InvalidFunction); } - pending.push((sec, r.off as u64, r.symbol.clone(), R_X86_64_64, 0)); + pending.push((sec, r.offset as u64, r.symbol.clone(), R_X86_64_64, 0)); } } for (section, offset, name, r_type, addend) in pending { @@ -139,7 +141,7 @@ mod tests { #[test] fn elf_function_section_and_symbol() { let mut f = Function::new(); - let t = f.types.parse_type("u64").unwrap(); + let t = f.types.parse_type("i64").unwrap(); let b = f.append_block(); let v = f.append_inst(b, t, Opcode::Iconst(42)); f.set_terminator(b, Terminator::Ret(Ret::one(v))); @@ -210,4 +212,54 @@ mod tests { .any(|s| s.name().unwrap() == "external_function" && s.is_undefined()) ); } + #[test] + fn elf_emits_globals_with_data_relocations() { + use volt_ir::module::DataReloc; + let mut f = Function::new(); + let t = f.types.parse_type("i64").unwrap(); + let b = f.append_block(); + let v = f.append_inst(b, t, Opcode::Iconst(7)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let mut module = Module::new(); + module.add_function("seven", f); + module.add_data("ro", alloc::vec![1, 2, 3, 4]); + module.add_writable_relocs( + "table", + alloc::vec![0; 16], + alloc::vec![DataReloc { + offset: 8, + symbol: "seven".into(), + }], + ); + module.add_bss("scratch", 64); + let bytes = write_module(&module).unwrap(); + let elf = object::File::parse(bytes.as_slice()).unwrap(); + assert_eq!( + elf.section_by_name(".rodata.ro").unwrap().data().unwrap(), + &[1, 2, 3, 4] + ); + assert_eq!(elf.section_by_name(".bss.scratch").unwrap().size(), 64); + let table = elf.section_by_name(".data.table").unwrap(); + assert_eq!(table.size(), 16); + let relocations: Vec<_> = table.relocations().collect(); + assert_eq!(relocations.len(), 1); + assert_eq!(relocations[0].0, 8); + assert_eq!( + relocations[0].1.flags(), + RelocationFlags::Elf { + r_type: object::elf::RelocationType(R_X86_64_64) + } + ); + // A relocation slot that does not fit is rejected. + let mut bad = Module::new(); + bad.add_data_relocs( + "short", + alloc::vec![0; 4], + alloc::vec![DataReloc { + offset: 0, + symbol: "seven".into(), + }], + ); + assert!(matches!(write_module(&bad), Err(Error::InvalidFunction))); + } } diff --git a/src/target/src/x86_64/tests.rs b/src/target/src/x86_64/tests.rs index 95c5a81..18e6ee4 100644 --- a/src/target/src/x86_64/tests.rs +++ b/src/target/src/x86_64/tests.rs @@ -1,5 +1,6 @@ // Ported from Vulcan native/cases/link test boundaries (Apache-2.0). // Copyright LilithSemi; Rust implementation modified for Volt. +use super::cases::{self, Val}; use super::*; use volt_ir::function::*; use volt_ir::types::Type; @@ -45,39 +46,117 @@ impl Drop for Executable { fn ty(f: &mut Function, s: &str) -> Type { f.types.parse_type(s).unwrap() } -fn binary(kind: &str, op: BinOp) -> Function { - let mut f = Function::new(); - let t = ty(&mut f, kind); - let b = f.append_block(); - let l = f.append_block_param(b, t); - let r = f.append_block_param(b, t); - let out = f.append_inst(b, t, Opcode::Arith(Arith { op, lhs: l, rhs: r })); - f.set_terminator(b, Terminator::Ret(Ret::one(out))); - f +/// Calls `code` with `args`; the result's raw register contents (an `f32` in the low 32 bits). +fn call(code: &Executable, args: &[Val], want: Val) -> u64 { + use Val::*; + let p = code.ptr; + // SAFETY: every transmute names the System V signature of the function that was compiled + // for exactly this argument/result shape. + unsafe { + match (args, want) { + ([Int(a)], Int(_)) => { + let f: unsafe extern "C" fn(u64) -> u64 = core::mem::transmute(p); + f(*a) + } + ([Int(a), Int(b)], Int(_)) => { + let f: unsafe extern "C" fn(u64, u64) -> u64 = core::mem::transmute(p); + f(*a, *b) + } + ([Int(a), Int(b), Int(c), Int(d)], Int(_)) => { + let f: unsafe extern "C" fn(u64, u64, u64, u64) -> u64 = core::mem::transmute(p); + f(*a, *b, *c, *d) + } + ([F64(a)], Int(_)) => { + let f: unsafe extern "C" fn(f64) -> u64 = core::mem::transmute(p); + f(*a) + } + ([F32(a)], Int(_)) => { + let f: unsafe extern "C" fn(f32) -> u64 = core::mem::transmute(p); + f(*a) + } + ([Int(a)], F64(_)) => { + let f: unsafe extern "C" fn(u64) -> f64 = core::mem::transmute(p); + f(*a).to_bits() + } + ([Int(a)], F32(_)) => { + let f: unsafe extern "C" fn(u64) -> f32 = core::mem::transmute(p); + f(*a).to_bits() as u64 + } + ([F32(a)], F64(_)) => { + let f: unsafe extern "C" fn(f32) -> f64 = core::mem::transmute(p); + f(*a).to_bits() + } + ([F64(a)], F32(_)) => { + let f: unsafe extern "C" fn(f64) -> f32 = core::mem::transmute(p); + f(*a).to_bits() as u64 + } + _ => panic!("no calling convention for {args:?} -> {want:?}"), + } + } +} +/// Every operation whose result depends on signedness, at every width, against the reference +/// arithmetic of `cases`: arguments carry garbage above their declared width and results must +/// come back exactly zero-extended. +#[test] +fn signedness_cases_match_the_reference() { + for case in cases::all() { + let code = Executable::new(&case.function); + for (args, want) in &case.runs { + assert_eq!( + call(&code, args, *want), + want.bits(), + "{} {args:x?}", + case.name + ); + } + } } #[test] fn scalar_division_shifts_and_high_product() { for (kind, op, a, b, want) in [ - ("u64", BinOp::Div, u64::MAX, 2, u64::MAX / 2), - ("u64", BinOp::Rem, 100, 9, 1), - ("u64", BinOp::Shr, 1 << 63, 63, 1), - ("u64", BinOp::Shl, 7, 65, 14), - ("u64", BinOp::Mulh, u64::MAX, u64::MAX, u64::MAX - 1), - ("i64", BinOp::Div, (-100i64) as u64, 9, (-11i64) as u64), - ("i64", BinOp::Mulh, (-2i64) as u64, 3, u64::MAX), + ("i64", BinOp::UDiv, u64::MAX, 2, u64::MAX / 2), + ("i64", BinOp::URem, 100, 9, 1), + ("i64", BinOp::Shr, 1 << 63, 63, 1), + ("i64", BinOp::Sar, 1 << 63, 63, u64::MAX), + ("i64", BinOp::Shl, 7, 65, 14), + ("i64", BinOp::UMulh, u64::MAX, u64::MAX, u64::MAX - 1), + ("i64", BinOp::SDiv, (-100i64) as u64, 9, (-11i64) as u64), + ("i64", BinOp::SRem, (-100i64) as u64, 9, (-1i64) as u64), + ( + "i64", + BinOp::UDiv, + (-100i64) as u64, + 9, + ((-100i64) as u64) / 9, + ), + ("i64", BinOp::SMulh, (-2i64) as u64, 3, u64::MAX), ] { - let mem = Executable::new(&binary(kind, op)); + let mem = Executable::new(&cases::arith(kind, op)); // SAFETY: the compiled function has two 64-bit integer parameters and one result. let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { core::mem::transmute(mem.ptr) }; assert_eq!(unsafe { entry(a, b) }, want, "{kind} {op:?}"); } } +/// The one narrow signed divide that does not trap: the most negative value divided by -1 +/// wraps, because the 32-bit hardware divide it runs on has room for the true quotient. +#[test] +fn narrow_signed_divide_of_minimum_by_minus_one_wraps() { + for (kind, min) in [("i8", 0x80u64), ("i16", 0x8000)] { + let neg_one = u64::MAX; + for (op, want) in [(BinOp::SDiv, min), (BinOp::SRem, 0)] { + let mem = Executable::new(&cases::arith(kind, op)); + let entry: unsafe extern "C" fn(u64, u64) -> u64 = + unsafe { core::mem::transmute(mem.ptr) }; + assert_eq!(unsafe { entry(min, neg_one) }, want, "{kind} {op:?}"); + } + } +} #[test] fn narrow_store_keeps_neighbor_bytes() { let mut f = Function::new(); let p = ty(&mut f, "ptr"); - let byte = ty(&mut f, "u8"); - let word = ty(&mut f, "u32"); + let byte = ty(&mut f, "i8"); + let word = ty(&mut f, "i32"); let b = f.append_block(); let ptr = f.append_block_param(b, p); let v = f.append_inst(b, byte, Opcode::Iconst(0xab)); @@ -87,7 +166,7 @@ fn narrow_store_keeps_neighbor_bytes() { word, Opcode::Load(Load { ptr, - volatile: false, + mem: MemFlags::new(), }), ); f.set_terminator(b, Terminator::Ret(Ret::one(r))); @@ -98,17 +177,255 @@ fn narrow_store_keeps_neighbor_bytes() { assert_eq!(bytes, [0xab, 0x22, 0x33, 0x44, 0x55]); } #[test] -fn signed_conversion_and_float_nan_compare() { +fn big_endian_loads_are_byte_swapped() { + let bytes: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; + for (kind, expected) in [ + ("i16", u16::from_be_bytes([bytes[0], bytes[1]]) as u64), + ( + "i32", + u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64, + ), + ("i64", u64::from_be_bytes(bytes)), + ] { + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, kind); + let wide = ty(&mut f, "i64"); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let mem = MemFlags { + endian: volt_ir::attribute::Endianness::Big, + ..MemFlags::new() + }; + let mut v = f.append_load(b, t, ptr, mem); + if t != wide { + v = f.append_inst( + b, + wide, + Opcode::Convert(Convert { + value: v, + kind: ConvertKind::Zext, + }), + ); + } + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*const u8) -> u64 = + unsafe { core::mem::transmute(code.ptr) }; + assert_eq!(unsafe { entry(bytes.as_ptr()) }, expected, "{kind}"); + } +} +/// `(p: ptr) -> i64` loading a `kind` integer `offset` bytes past `p` with `mem` and widening +/// it: sign-extending when `signed`, zero-extending otherwise (a load itself never extends +/// with the sign). +fn ordered_load(kind: &str, signed: bool, mem: MemFlags, offset: i64) -> Function { + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, kind); + let w = ty(&mut f, "i64"); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let addr = if offset == 0 { + ptr + } else { + f.append_arith_imm(b, p, BinOp::Add, ptr, offset) + }; + let mut v = f.append_load(b, t, addr, mem); + if t != w { + v = f.append_inst( + b, + w, + Opcode::Convert(Convert { + value: v, + kind: if signed { + ConvertKind::Sext + } else { + ConvertKind::Zext + }, + }), + ); + } + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + f +} +/// `(p: ptr, x: i64)` storing the low bits of `x` as a `kind` integer `offset` bytes past `p`. +fn ordered_store(kind: &str, mem: MemFlags, offset: i64) -> Function { let mut f = Function::new(); - let i = ty(&mut f, "i32"); - let o = ty(&mut f, "i64"); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, kind); + let w = ty(&mut f, "i64"); let b = f.append_block(); - let v = f.append_block_param(b, i); - let r = f.append_inst(b, o, Opcode::Convert(Convert { value: v })); - f.set_terminator(b, Terminator::Ret(Ret::one(r))); - let mem = Executable::new(&f); - let entry: unsafe extern "C" fn(i32) -> i64 = unsafe { core::mem::transmute(mem.ptr) }; - assert_eq!(unsafe { entry(-42) }, -42); + let ptr = f.append_block_param(b, p); + let x = f.append_block_param(b, w); + let narrow = if t == w { + x + } else { + f.append_inst( + b, + t, + Opcode::Convert(Convert { + value: x, + kind: ConvertKind::Trunc, + }), + ) + }; + let addr = if offset == 0 { + ptr + } else { + f.append_arith_imm(b, p, BinOp::Add, ptr, offset) + }; + f.append_store_mem(b, narrow, addr, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + f +} +fn ordering_mem(ordering: Option, bytes: u32) -> MemFlags { + MemFlags { + ordering, + align: bytes, + ..MemFlags::new() + } +} +const LOAD_ORDERINGS: [Option; 4] = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Acquire), + Some(AtomicOrdering::SeqCst), +]; +const STORE_ORDERINGS: [Option; 4] = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Release), + Some(AtomicOrdering::SeqCst), +]; +#[repr(align(16))] +struct Image([u8; 16]); +const IMAGE: Image = Image([ + 0x81, 0x92, 0xa3, 0xb4, 0xc5, 0xd6, 0xe7, 0xf8, 0x09, 0x1a, 0x2b, 0x3c, 0x4d, 0x5e, 0x6f, 0x70, +]); +/// Every width and ordering loads the same value an ordinary load would, zero- or +/// sign-extended as the explicit conversion says. +#[test] +fn ordered_loads_produce_the_plain_load_value() { + let image = IMAGE; + for bits in [8u32, 16, 32, 64] { + for signed in [false, true] { + if signed && bits == 64 { + continue; + } + for offset in [0usize, 8] { + let n = (bits / 8) as usize; + let mut raw = [0u8; 8]; + raw[..n].copy_from_slice(&image.0[offset..offset + n]); + let zero = u64::from_le_bytes(raw); + let expected = if signed { + (((zero << (64 - bits)) as i64) >> (64 - bits)) as u64 + } else { + zero + }; + for ordering in LOAD_ORDERINGS { + let f = ordered_load( + &format!("i{bits}"), + signed, + ordering_mem(ordering, bits / 8), + offset as i64, + ); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*const u8) -> u64 = + unsafe { core::mem::transmute(code.ptr) }; + assert_eq!( + unsafe { entry(image.0.as_ptr()) }, + expected, + "i{bits} signed={signed} offset={offset} {ordering:?}" + ); + } + } + } + } +} +/// Every width and ordering (including the `xchg` of a sequentially consistent store) writes +/// exactly its own bytes. +#[test] +fn ordered_stores_write_exactly_their_width() { + let x = 0x0123_4567_89ab_cdefu64; + for (kind, bits) in [("i8", 8u32), ("i16", 16), ("i32", 32), ("i64", 64)] { + for offset in [0usize, 8] { + let n = (bits / 8) as usize; + for ordering in STORE_ORDERINGS { + let f = ordered_store(kind, ordering_mem(ordering, bits / 8), offset as i64); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*mut u8, u64) = + unsafe { core::mem::transmute(code.ptr) }; + let mut out = Image([0xee; 16]); + unsafe { entry(out.0.as_mut_ptr(), x) }; + let mut want = [0xeeu8; 16]; + want[offset..offset + n].copy_from_slice(&x.to_le_bytes()[..n]); + assert_eq!(out.0, want, "{kind} offset={offset} {ordering:?}"); + } + } + } +} +/// Float and pointer accesses: atomic floats go through an integer register and pointers are +/// 64-bit integers. +#[test] +fn ordered_float_and_pointer_accesses_execute() { + for ordering in [AtomicOrdering::Acquire, AtomicOrdering::SeqCst] { + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, "f64"); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let v = f.append_load(b, t, ptr, ordering_mem(Some(ordering), 8)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*const f64) -> f64 = + unsafe { core::mem::transmute(code.ptr) }; + let x = 6.25f64; + assert_eq!(unsafe { entry(&x) }, 6.25); + } + for ordering in [AtomicOrdering::Release, AtomicOrdering::SeqCst] { + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, "f32"); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let x = f.append_block_param(b, t); + f.append_store_mem(b, x, ptr, ordering_mem(Some(ordering), 4)); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*mut f32, f32) = unsafe { core::mem::transmute(code.ptr) }; + let mut out = 0.0f32; + unsafe { entry(&mut out, -2.75) }; + assert_eq!(out, -2.75); + } + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let b = f.append_block(); + let src = f.append_block_param(b, p); + let dst = f.append_block_param(b, p); + let v = f.append_load(b, p, src, ordering_mem(Some(AtomicOrdering::Acquire), 8)); + f.append_store_mem(b, v, dst, ordering_mem(Some(AtomicOrdering::SeqCst), 8)); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*const *const u8, *mut *const u8) = + unsafe { core::mem::transmute(code.ptr) }; + let target = 0x1234usize as *const u8; + let mut out = core::ptr::null(); + unsafe { entry(&target, &mut out) }; + assert_eq!(out, target); +} +#[test] +fn sign_and_zero_extension_differ() { + for (kind, want) in [ + (ConvertKind::Sext, -42i64 as u64), + (ConvertKind::Zext, 0xffff_ffd6), + ] { + let mem = Executable::new(&cases::convert("i32", "i64", kind)); + let entry: unsafe extern "C" fn(i32) -> u64 = unsafe { core::mem::transmute(mem.ptr) }; + assert_eq!(unsafe { entry(-42) }, want, "{kind:?}"); + } +} +#[test] +fn float_compare_with_nan_is_unordered() { for (cmp, want) in [ (CmpOp::Eq, 0), (CmpOp::Ne, 1), @@ -135,13 +452,13 @@ fn signed_conversion_and_float_nan_compare() { f.set_terminator(b, Terminator::Ret(Ret::one(v))); let mem = Executable::new(&f); let entry: unsafe extern "C" fn(f64, f64) -> u64 = unsafe { core::mem::transmute(mem.ptr) }; - assert_eq!(unsafe { entry(f64::NAN, 1.0) }, want); + assert_eq!(unsafe { entry(f64::NAN, 1.0) }, want, "{cmp:?}"); } } #[test] fn cfg_edge_parameter_cycles() { let mut f = Function::new(); - let t = ty(&mut f, "u64"); + let t = ty(&mut f, "i64"); let boolean = ty(&mut f, "bool"); let entry = f.append_block(); let loop_b = f.append_block(); @@ -193,7 +510,7 @@ fn vector_reduce_expansion() { v, Opcode::Load(Load { ptr, - volatile: false, + mem: MemFlags::new(), }), ); let r = f.append_reduce(b, BinOp::Add, loaded); @@ -208,7 +525,7 @@ fn indirect_call_preserves_live_values() { x * 3 } let mut f = Function::new(); - let t = ty(&mut f, "u64"); + let t = ty(&mut f, "i64"); let p = ty(&mut f, "ptr"); let b = f.append_block(); let x = f.append_block_param(b, t); @@ -242,18 +559,70 @@ fn indirect_call_preserves_live_values() { let entry: unsafe extern "C" fn(u64) -> u64 = unsafe { core::mem::transmute(mem.ptr) }; assert_eq!(unsafe { entry(13) }, 52); } +core::arch::global_asm!( + ".globl volt_test_dirty_return", + "volt_test_dirty_return:", + // al = 0x80, everything above it set: the System V ABI leaves those bits undefined. + "mov rax, -128", + "ret", +); +unsafe extern "C" { + fn volt_test_dirty_return() -> u8; +} +/// The bits above a narrow integer a callee returns are undefined, so a call result is +/// canonicalized (zero-extended from its declared width) before anything reads it. +#[test] +fn narrow_call_results_are_zero_extended_from_their_width() { + for (kind, convert, want) in [ + (ConvertKind::Zext, "i64", 0x80u64), + (ConvertKind::Sext, "i64", (-128i64) as u64), + (ConvertKind::Zext, "i32", 0x80), + (ConvertKind::Sext, "i32", 0xffff_ff80), + ] { + let mut f = Function::new(); + let byte = ty(&mut f, "i8"); + let wide = ty(&mut f, convert); + let p = ty(&mut f, "ptr"); + let b = f.append_block(); + let target = f.append_inst( + b, + p, + Opcode::Iconst(volt_test_dirty_return as *const () as usize as i64), + ); + let args = f.intern_values(&[]); + let r = f.append_inst( + b, + byte, + Opcode::CallIndirect(CallIndirect { + target, + args, + is_variadic: false, + num_fixed: 0, + ret_dest: None, + ret_regs: 0, + ret_pieces: [RetPiece::default(); 4], + sret: false, + }), + ); + let w = f.append_inst(b, wide, Opcode::Convert(Convert { value: r, kind })); + f.set_terminator(b, Terminator::Ret(Ret::one(w))); + let mem = Executable::new(&f); + let entry: unsafe extern "C" fn() -> u64 = unsafe { core::mem::transmute(mem.ptr) }; + assert_eq!(unsafe { entry() }, want, "{kind:?} to {convert}"); + } +} #[test] fn softfp_relocation_owns_injected_symbol() { - let f = binary("f128", BinOp::Add); + let f = cases::arith("f128", BinOp::Add); let out = isel::compile_object(&f).unwrap(); assert!(out.relocs.iter().any(|r| r.symbol == "__addtf3")); assert_eq!(f.symbol_count(), 0); } #[test] -fn narrow_arithmetic_wraps_and_compares_signed_values() { +fn narrow_arithmetic_wraps_and_compares_by_operation() { let mut f = Function::new(); - let byte = ty(&mut f, "u8"); + let byte = ty(&mut f, "i8"); let boolean = ty(&mut f, "bool"); let block = f.append_block(); let input = f.append_block_param(block, byte); @@ -283,46 +652,62 @@ fn narrow_arithmetic_wraps_and_compares_signed_values() { assert_eq!(unsafe { entry(255) }, 1); assert_eq!(unsafe { entry(1) }, 0); - let mut f = Function::new(); - let signed = ty(&mut f, "i8"); - let boolean = ty(&mut f, "bool"); - let block = f.append_block(); - let minus_one = f.append_inst(block, signed, Opcode::Iconst(-1)); - let zero = f.append_inst(block, signed, Opcode::Iconst(0)); - let negative = f.append_inst( - block, - boolean, - Opcode::Icmp(Compare { - op: CmpOp::Lt, - lhs: minus_one, - rhs: zero, - }), - ); - f.set_terminator(block, Terminator::Ret(Ret::one(negative))); - let memory = Executable::new(&f); - let entry: unsafe extern "C" fn() -> u64 = unsafe { core::mem::transmute(memory.ptr) }; - assert_eq!(unsafe { entry() }, 1); + // 0xff is -1 as a signed byte (less than 0) and 255 as an unsigned one (not less). + for (op, want) in [ + (CmpOp::Slt, 1), + (CmpOp::Ult, 0), + (CmpOp::Sgt, 0), + (CmpOp::Ugt, 1), + ] { + let mut f = Function::new(); + let byte = ty(&mut f, "i8"); + let boolean = ty(&mut f, "bool"); + let block = f.append_block(); + let minus_one = f.append_inst(block, byte, Opcode::Iconst(-1)); + let zero = f.append_inst(block, byte, Opcode::Iconst(0)); + let result = f.append_inst( + block, + boolean, + Opcode::Icmp(Compare { + op, + lhs: minus_one, + rhs: zero, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(result))); + let memory = Executable::new(&f); + let entry: unsafe extern "C" fn() -> u64 = unsafe { core::mem::transmute(memory.ptr) }; + assert_eq!(unsafe { entry() }, want, "{op:?}"); + } } #[test] fn high_product_uses_the_declared_integer_width() { - for (kind, a, b, expected) in [ - ("u8", 255, 255, 254), - ("u16", 65535, 65535, 65534), - ("u48", (1u64 << 48) - 1, (1u64 << 48) - 1, (1u64 << 48) - 2), - ("i8", (-2i64) as u64, 3, 255), + for (kind, op, a, b, expected) in [ + ("i8", BinOp::UMulh, 255, 255, 254), + ("i16", BinOp::UMulh, 65535, 65535, 65534), + ( + "i48", + BinOp::UMulh, + (1u64 << 48) - 1, + (1u64 << 48) - 1, + (1u64 << 48) - 2, + ), + // 0xfe is -2: the signed high byte of -2 * 3 = -6 is -1, the unsigned one of 254 * 3 = 762 + // is 2. + ("i8", BinOp::SMulh, 0xfe, 3, 0xff), + ("i8", BinOp::UMulh, 0xfe, 3, 2), + ("i16", BinOp::SMulh, 0xfffe, 3, 0xffff), + ("i16", BinOp::UMulh, 0xfffe, 3, 2), + ("i32", BinOp::SMulh, 0xffff_fffe, 3, 0xffff_ffff), + ("i32", BinOp::UMulh, 0xffff_fffe, 3, 2), + ("i48", BinOp::SMulh, (1u64 << 48) - 2, 3, (1u64 << 48) - 1), + ("i48", BinOp::UMulh, (1u64 << 48) - 2, 3, 2), ] { - let memory = Executable::new(&binary(kind, BinOp::Mulh)); + let memory = Executable::new(&cases::arith(kind, op)); let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { core::mem::transmute(memory.ptr) }; - let mask = if kind == "u48" { - (1u64 << 48) - 1 - } else if kind == "u16" { - 65535 - } else { - 255 - }; - assert_eq!(unsafe { entry(a, b) } & mask, expected, "{kind}"); + assert_eq!(unsafe { entry(a, b) }, expected, "{kind} {op:?}"); } } @@ -334,7 +719,7 @@ fn sret_returns_the_original_hidden_pointer_after_its_last_explicit_use() { let mut f = Function::new(); f.sret = true; let pointer = ty(&mut f, "ptr"); - let word = ty(&mut f, "u64"); + let word = ty(&mut f, "i64"); let block = f.append_block(); let destination = f.append_block_param(block, pointer); let value = f.append_inst(block, word, Opcode::Iconst(7)); From 17e055b44224037dd4ca92cbb18b8cdf2190f787 Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Fri, 9 Oct 2026 16:33:01 -0700 Subject: [PATCH 11/13] feat: add volt aarch64 lifter --- Cargo.lock | 9 + Cargo.toml | 2 + README.md | 10 + src/emulator/Cargo.toml | 1 + src/emulator/src/aarch64/compile.rs | 3154 +---------------------- src/emulator/src/aarch64/environment.rs | 1037 ++++++++ src/emulator/src/aarch64/mod.rs | 1 + src/lift/aarch64/Cargo.toml | 15 + src/lift/aarch64/src/builder.rs | 284 ++ src/lift/aarch64/src/int.rs | 383 +++ src/lift/aarch64/src/lib.rs | 516 ++++ src/lift/aarch64/src/simd.rs | 859 ++++++ src/lift/aarch64/src/tests.rs | 949 +++++++ 13 files changed, 4088 insertions(+), 3132 deletions(-) create mode 100644 src/emulator/src/aarch64/environment.rs create mode 100644 src/lift/aarch64/Cargo.toml create mode 100644 src/lift/aarch64/src/builder.rs create mode 100644 src/lift/aarch64/src/int.rs create mode 100644 src/lift/aarch64/src/lib.rs create mode 100644 src/lift/aarch64/src/simd.rs create mode 100644 src/lift/aarch64/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 6c9ce70..6113cd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,6 +179,7 @@ dependencies = [ "libc", "parking_lot", "volt-ir", + "volt-lift-aarch64", "volt-target", ] @@ -195,6 +196,14 @@ dependencies = [ name = "volt-ir" version = "0.1.0" +[[package]] +name = "volt-lift-aarch64" +version = "0.1.0" +dependencies = [ + "volt-ir", + "volt-target", +] + [[package]] name = "volt-target" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index fb282c3..79238b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "src/ir", "src/emulator", "src/emulator/cli", + "src/lift/aarch64", ] [workspace.package] @@ -16,6 +17,7 @@ license = "Apache-2.0" volt-ir = { path = "src/ir" } volt-target = { path = "src/target" } volt-emulator = { path = "src/emulator" } +volt-lift-aarch64 = { path = "src/lift/aarch64" } object = { version = "0.40.0", default-features = false, features = ["read_core", "write_core", "elf", "macho"] } libc = "0.2" diff --git a/README.md b/README.md index a7631f3..25abf59 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ Independent Rust workspace for modular dynamic binary translation. +## AArch64 lifter + +`volt-lift-aarch64` (`src/lift/aarch64`) is a `no_std` crate that lifts decoded AArch64 +instructions to `volt-ir` over guest state held in memory: each lifted function is +`fn(state: *mut u8) -> u64` and returns the next guest pc. It implements the architecture +(integer ALU, flags, branches, register-only SIMD) and leaves the machine to an +`Environment` trait: the state layout, how memory is reached, and system instructions. +`volt-emulator` is one environment (`Cpu` state, an inline data TLB, trap-and-resume slow +paths); a user-mode emulator or a static translator supplies its own. + ## AArch64 Linux boot `volt-boot` launches a raw arm64 Linux `Image` with a PL011 console, GICv2, diff --git a/src/emulator/Cargo.toml b/src/emulator/Cargo.toml index 1e4501d..d48a800 100644 --- a/src/emulator/Cargo.toml +++ b/src/emulator/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true [dependencies] volt-ir.workspace = true volt-target.workspace = true +volt-lift-aarch64.workspace = true libc = { workspace = true, optional = true } parking_lot = { workspace = true, optional = true } diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index c9300c9..98dbc99 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -1,18 +1,11 @@ use super::{ - cpu::{Cpu, DTLB_ENTRIES, Exclusive as CpuExclusive, System as CpuSystem, Trap}, + cpu::{Cpu, Trap}, + environment::CpuEnvironment, host::{Dispatch, dispatch}, }; use crate::memory::MemoryError; -use core::{ - cell::{Cell, RefCell}, - fmt, - mem::offset_of, -}; -use volt_ir::{ - function::{self as ir, BinOp as B, CmpOp as C, Function, Opcode, Value}, - types::{IntDesc, Type, TypeKind}, -}; -use volt_target::aarch64::decode::*; +use core::fmt; +use volt_lift_aarch64::{LiftError, Lifter}; use volt_target::native; #[derive(Debug)] @@ -25,6 +18,7 @@ pub enum Error { UnsupportedInstruction, AddressOverflow, InvalidInstructionLength, + Lift(LiftError), } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -39,6 +33,7 @@ impl fmt::Display for Error { Self::UnsupportedInstruction => f.write_str("unsupported instruction encoding"), Self::AddressOverflow => f.write_str("instruction address overflow"), Self::InvalidInstructionLength => f.write_str("invalid instruction fetch length"), + Self::Lift(e) => write!(f, "{e}"), } } } @@ -48,6 +43,11 @@ impl From for Error { Self::Memory(e) } } +impl From for Error { + fn from(e: LiftError) -> Self { + Self::Lift(e) + } +} impl From for Error { fn from(e: native::Error) -> Self { Self::Native(e) @@ -76,555 +76,6 @@ impl Block { next } } -struct Lower { - f: RefCell, - /// Block currently being emitted into. Memory fast paths switch it. - block: Cell, - cpu: Value, - i64: Type, - i32: Type, - i8: Type, - boolean: Type, - ptr: Type, - /// Guest instructions started but not yet added to `cntvct_el0`. The - /// counter is bumped once per exit path (`ret`) and before anything that - /// reads it, instead of with a load/add/store per instruction. - pending: Cell, -} -impl Lower { - fn new() -> Self { - let mut f = Function::new(); - // A typical block is a few blocks and about a hundred instructions. - f.reserve(8, 128, 128); - let i64 = f.types.intern(TypeKind::Int(IntDesc { bits: 64 })); - let i32 = f.types.intern(TypeKind::Int(IntDesc { bits: 32 })); - let i8 = f.types.intern(TypeKind::Int(IntDesc { bits: 8 })); - let boolean = f.types.intern(TypeKind::Bool); - let ptr = f.types.ptr_global(); - let block = f.append_block(); - let cpu = f.append_block_param(block, ptr); - Self { - f: RefCell::new(f), - block: Cell::new(block), - cpu, - i64, - i32, - i8, - boolean, - ptr, - pending: Cell::new(0), - } - } - fn emit(&self, t: Type, op: Opcode) -> Value { - self.f.borrow_mut().append_inst(self.block.get(), t, op) - } - fn k(&self, t: Type, n: u64) -> Value { - self.emit(t, Opcode::Iconst(n as i64)) - } - fn bin(&self, t: Type, op: B, a: Value, b: Value) -> Value { - self.emit(t, Opcode::Arith(ir::Arith { op, lhs: a, rhs: b })) - } - fn imm(&self, t: Type, op: B, a: Value, n: u64) -> Value { - // `ArithImm` instead of `Arith` over an `Iconst`: the backend encodes - // small immediates directly and `addrfold` only folds `ArithImm` adds - // into load/store addressing. - self.emit( - t, - Opcode::ArithImm(ir::ArithImm { - op, - lhs: a, - imm: n as i64, - }), - ) - } - fn cv(&self, t: Type, v: Value) -> Value { - let from = self.f.borrow().value_type(v); - if from == t { - return v; - } - let kind = { - let f = self.f.borrow(); - let bits = |ty| match f.types.type_kind(ty) { - TypeKind::Int(d) => d.bits, - _ => 1, - }; - if bits(t) > bits(from) { - ir::ConvertKind::Zext - } else { - ir::ConvertKind::Trunc - } - }; - self.emit(t, Opcode::Convert(ir::Convert { value: v, kind })) - } - fn cmp(&self, op: C, a: Value, b: Value) -> Value { - self.emit( - self.boolean, - Opcode::Icmp(ir::Compare { op, lhs: a, rhs: b }), - ) - } - fn sel(&self, t: Type, c: Value, a: Value, b: Value) -> Value { - self.emit( - t, - Opcode::Select(ir::Select { - cond: c, - then: a, - else_: b, - }), - ) - } - fn addr(&self, at: usize) -> Value { - self.imm(self.ptr, B::Add, self.cpu, at as u64) - } - fn load_ptr(&self, t: Type, p: Value) -> Value { - self.emit( - t, - Opcode::Load(ir::Load { - ptr: p, - mem: ir::MemFlags::default(), - }), - ) - } - fn load(&self, t: Type, at: usize) -> Value { - self.load_ptr(t, self.addr(at)) - } - fn store_ptr(&self, p: Value, v: Value) { - self.f.borrow_mut().append_store(self.block.get(), v, p) - } - fn store(&self, at: usize, v: Value) { - self.store_ptr(self.addr(at), v) - } - fn byte(&self, at: usize, v: u64) { - self.store(at, self.k(self.i8, v)) - } - fn reg_at(r: u8) -> usize { - if r == 31 { - offset_of!(Cpu, sp) - } else { - offset_of!(Cpu, x) + r as usize * 8 - } - } - fn reg(&self, t: Type, r: u8, zero: bool) -> Value { - if zero && r == 31 { - self.k(t, 0) - } else { - self.cv(t, self.load(self.i64, Self::reg_at(r))) - } - } - fn put(&self, r: u8, v: Value) { - if r != 31 { - self.put_sp(r, v) - } - } - fn put_sp(&self, r: u8, v: Value) { - self.store(Self::reg_at(r), self.cv(self.i64, v)) - } - fn ty(&self, w: Width) -> Type { - if w == Width::X64 { self.i64 } else { self.i32 } - } - fn add_counter(&self, n: u64) { - let at = offset_of!(Cpu, system) + offset_of!(CpuSystem, cntvct_el0); - self.store(at, self.imm(self.i64, B::Add, self.load(self.i64, at), n)); - } - /// Make `cntvct_el0` in memory current on the path being emitted. - fn flush_counter(&self) { - let n = self.pending.replace(0); - if n != 0 { - self.add_counter(n); - } - } - fn ret(&self, v: Value) { - // Every exit leaves with the counter current. `pending` is not - // cleared: sibling paths (slow blocks) share the main path's count. - let n = self.pending.get(); - if n != 0 { - self.add_counter(n); - } - self.f - .borrow_mut() - .set_terminator(self.block.get(), ir::Terminator::Ret(ir::Ret::one(v))) - } - fn not(&self, v: Value) -> Value { - self.imm(self.boolean, B::BitXor, v, 1) - } - fn flag(&self, flags: Value, bit: u32) -> Value { - self.cmp( - C::Ne, - self.imm(self.i32, B::BitAnd, flags, 1 << bit), - self.k(self.i32, 0), - ) - } - fn condition(&self, c: Condition) -> Value { - let f = self.load(self.i32, offset_of!(Cpu, flags)); - let n = self.flag(f, 31); - let z = self.flag(f, 30); - let carry = self.flag(f, 29); - let v = self.flag(f, 28); - let changed = self.bin(self.boolean, B::BitXor, n, v); - let ordered = self.not(changed); - let positive = self.not(z); - match c { - Condition::Eq => z, - Condition::Ne => positive, - Condition::Cs => carry, - Condition::Cc => self.not(carry), - Condition::Mi => n, - Condition::Pl => self.not(n), - Condition::Vs => v, - Condition::Vc => self.not(v), - Condition::Hi => self.bin(self.boolean, B::BitAnd, carry, positive), - Condition::Ls => self.not(self.bin(self.boolean, B::BitAnd, carry, positive)), - Condition::Ge => ordered, - Condition::Lt => changed, - Condition::Gt => self.bin(self.boolean, B::BitAnd, positive, ordered), - Condition::Le => self.not(self.bin(self.boolean, B::BitAnd, positive, ordered)), - Condition::Al | Condition::Nv => self.k(self.boolean, 1), - } - } - fn pack(&self, n: Value, z: Value, c: Value, v: Value) -> Value { - let mut out = self.k(self.i32, 0); - for (bit, shift) in [(n, 31), (z, 30), (c, 29), (v, 28)] { - out = self.bin( - self.i32, - B::BitOr, - out, - self.imm(self.i32, B::Shl, self.cv(self.i32, bit), shift), - ); - } - out - } - fn flags( - &self, - w: Width, - op: ArithOp, - a: Value, - b: Value, - r: Value, - carry: Option, - ) -> Value { - let t = self.ty(w); - let zero = self.k(t, 0); - let n = self.cmp( - C::Ne, - self.imm(t, B::BitAnd, r, 1u64 << ((w as u32) - 1)), - zero, - ); - let z = self.cmp(C::Eq, r, zero); - let c = carry.unwrap_or_else(|| { - if op == ArithOp::Sub { - self.cmp(C::Uge, a, b) - } else { - self.cmp(C::Ult, r, a) - } - }); - let an = self.cmp(C::Slt, a, zero); - let bn = self.cmp(C::Slt, b, zero); - let rn = self.cmp(C::Slt, r, zero); - let diff = self.bin(self.boolean, B::BitXor, an, bn); - let flip = self.bin(self.boolean, B::BitXor, an, rn); - let v = self.bin( - self.boolean, - B::BitAnd, - if op == ArithOp::Sub { - diff - } else { - self.not(diff) - }, - flip, - ); - self.pack(n, z, c, v) - } - fn logic_flags(&self, w: Width, r: Value) { - let t = self.ty(w); - let zero = self.k(t, 0); - let n = self.cmp( - C::Ne, - self.imm(t, B::BitAnd, r, 1u64 << ((w as u32) - 1)), - zero, - ); - let z = self.cmp(C::Eq, r, zero); - let no = self.k(self.boolean, 0); - self.store(offset_of!(Cpu, flags), self.pack(n, z, no, no)); - } - fn shift(&self, w: Width, v: Value, s: Shift, n: u64) -> Value { - if n == 0 { - return v; - } - let t = self.ty(w); - match s { - Shift::Lsl => self.imm(t, B::Shl, v, n), - Shift::Lsr => self.imm(t, B::Shr, v, n), - Shift::Asr => self.imm(t, B::Sar, v, n), - Shift::Ror => self.bin( - t, - B::BitOr, - self.imm(t, B::Shr, v, n), - self.imm(t, B::Shl, v, w as u64 - n), - ), - } - } - fn extend(&self, v: Value, e: Extend) -> Value { - match e { - Extend::Uxtx | Extend::Sxtx => v, - Extend::Uxtb => self.imm(self.i64, B::BitAnd, v, 0xff), - Extend::Uxth => self.imm(self.i64, B::BitAnd, v, 0xffff), - Extend::Uxtw => self.imm(self.i64, B::BitAnd, v, 0xffffffff), - Extend::Sxtb | Extend::Sxth | Extend::Sxtw => { - let n = match e { - Extend::Sxtb => 56, - Extend::Sxth => 48, - _ => 32, - }; - let raised = self.imm(self.i64, B::Shl, v, n); - self.shift(Width::X64, raised, Shift::Asr, n) - } - } - } - fn address(&self, rn: u8, a: Addressing) -> Value { - let base = self.reg(self.i64, rn, false); - match a { - Addressing::Offset(n) => self.imm(self.i64, B::Add, base, n as u64), - Addressing::PreIndex(n) => { - let a = self.imm(self.i64, B::Add, base, n as u64); - self.put_sp(rn, a); - a - } - Addressing::PostIndex(n) => { - self.store( - offset_of!(Cpu, writeback_value), - self.imm(self.i64, B::Add, base, n as u64), - ); - self.byte(offset_of!(Cpu, writeback_dest), rn as u64); - self.byte(offset_of!(Cpu, writeback), 1); - base - } - Addressing::Register { rm, extend, amount } => { - let v = self.extend(self.reg(self.i64, rm, true), extend); - self.bin( - self.i64, - B::Add, - base, - self.imm(self.i64, B::Shl, v, amount as u64), - ) - } - } - } - fn branch(&self, pc: u64, offset: i64, c: Value) { - self.ret(self.sel( - self.i64, - c, - self.k(self.i64, pc.wrapping_add(offset as u64)), - self.k(self.i64, pc.wrapping_add(4)), - )) - } - fn int(&self, bits: u16) -> Type { - self.f - .borrow_mut() - .types - .intern(TypeKind::Int(IntDesc { bits })) - } - fn new_block(&self) -> ir::Block { - self.f.borrow_mut().append_block() - } - fn branch_if(&self, c: Value, then: ir::Block, otherwise: ir::Block) { - self.f.borrow_mut().append_if( - self.block.get(), - c, - ir::EdgeDesc::bare(then), - ir::EdgeDesc::bare(otherwise), - ) - } - /// Like `address`, but a base-register writeback (pre- or post-index) is - /// returned instead of applied or scheduled for the machine, so each path - /// can apply it its own way once the access has succeeded. An access that - /// faults must leave the base register untouched so the retry sees it. - fn address_split(&self, rn: u8, a: Addressing) -> (Value, Option<(u8, Value)>) { - match a { - Addressing::PostIndex(n) => { - let base = self.reg(self.i64, rn, false); - (base, Some((rn, self.imm(self.i64, B::Add, base, n as u64)))) - } - Addressing::PreIndex(n) => { - let base = self.reg(self.i64, rn, false); - let address = self.imm(self.i64, B::Add, base, n as u64); - (address, Some((rn, address))) - } - other => (self.address(rn, other), None), - } - } - /// Probe the inline data TLB for `address`. Returns `(hit, host_pointer)`; - /// the pointer is only meaningful when `hit`. A hit requires the page to be - /// cached for this access kind and the access to be naturally aligned (so it - /// cannot cross a page), because the tag compare covers the low `size - 1` - /// address bits too. - fn dtlb_probe(&self, address: Value, size: Size, write: bool) -> (Value, Value) { - let n = size as u64; - // Entry offset = ((address >> 12) & (ENTRIES - 1)) * 32 = (address >> 7) & mask. - let index = self.imm( - self.i64, - B::BitAnd, - self.imm(self.i64, B::Shr, address, 7), - (DTLB_ENTRIES as u64 - 1) << 5, - ); - let entry = self.bin(self.ptr, B::Add, self.cpu, index); - // Constant displacements fold into the load addressing. - let field = |at: usize| { - self.load_ptr( - self.i64, - self.imm(self.ptr, B::Add, entry, (offset_of!(Cpu, dtlb) + at) as u64), - ) - }; - let tag = field(if write { 8 } else { 0 }); - let addend = field(16); - let host = self.bin(self.ptr, B::Add, address, addend); - let want = self.imm(self.i64, B::BitAnd, address, !0xfffu64 | (n - 1)); - // Last, so the backend can fuse it into the branch that consumes it. - let hit = self.cmp(C::Eq, tag, want); - (hit, host) - } - fn fast_load(&self, host: Value, size: Size, rt: u8, signed: SignExtend) { - let bits = size as u16 * 8; - let raw = self.load_ptr(self.int(bits), host); - if rt == 31 { - return; - } - let mut v = self.cv(self.i64, raw); - if signed != SignExtend::None && bits < 64 { - let n = 64 - bits as u64; - let extended = self.shift(Width::X64, self.imm(self.i64, B::Shl, v, n), Shift::Asr, n); - v = if signed == SignExtend::To32 { - self.imm(self.i64, B::BitAnd, extended, 0xffff_ffff) - } else { - extended - }; - } - self.put(rt, v); - } - fn fast_store(&self, host: Value, size: Size, rt: u8) { - let t = self.int(size as u16 * 8); - self.store_ptr(host, self.cv(t, self.reg(self.i64, rt, true))); - } - /// Emit a load or store with an inline fast path. On a data-TLB hit the - /// access happens in place and emission continues in a fresh block; on a - /// miss control goes to a slow block that hands the access to the machine - /// (`slow` fills in the request) and returns. - fn memory_access( - &self, - address: Value, - size: Size, - op: MemoryOp, - rt: u8, - signed: SignExtend, - post: Option<(u8, Value)>, - slow: impl FnOnce(&Lower), - ) { - let write = op == MemoryOp::Store; - let (hit, host) = self.dtlb_probe(address, size, write); - let (fast_block, slow_block) = (self.new_block(), self.new_block()); - self.branch_if(hit, fast_block, slow_block); - self.block.set(slow_block); - slow(self); - self.block.set(fast_block); - if write { - self.fast_store(host, size, rt); - } else { - self.fast_load(host, size, rt, signed); - } - if let Some((rn, value)) = post { - self.put_sp(rn, value); - } - } - /// Inline fast path for a GPR load/store pair: one data-TLB probe covers - /// both accesses when the first is naturally aligned and the second starts - /// in the same page. Anything else takes `slow`, which hands the whole pair - /// to the machine. - fn pair_access( - &self, - address: Value, - a: Pair, - post: Option<(u8, Value)>, - slow: impl FnOnce(&Lower), - ) { - let write = a.op == MemoryOp::Store; - let n = a.size as u64; - let (hit, host) = self.dtlb_probe(address, a.size, write); - let (check_block, fast_block, slow_block) = - (self.new_block(), self.new_block(), self.new_block()); - self.branch_if(hit, check_block, slow_block); - // `address` is aligned to `n`, so the second access leaves the page - // exactly when it starts on a page boundary. - self.block.set(check_block); - let second_at = self.imm(self.i64, B::Add, address, n); - let offset = self.imm(self.i64, B::BitAnd, second_at, 0xfff); - let same_page = self.cmp(C::Ne, offset, self.k(self.i64, 0)); - self.branch_if(same_page, fast_block, slow_block); - self.block.set(slow_block); - slow(self); - self.block.set(fast_block); - let second = self.imm(self.ptr, B::Add, host, n); - if write { - self.fast_store(host, a.size, a.rt); - self.fast_store(second, a.size, a.rt2); - } else { - self.fast_load(host, a.size, a.rt, a.signed); - self.fast_load(second, a.size, a.rt2, a.signed); - } - if let Some((rn, value)) = post { - self.put_sp(rn, value); - } - } - fn trap(&self, pc: u64, t: Trap) { - self.byte(offset_of!(Cpu, trap), t as u64); - self.ret(self.k(self.i64, pc)) - } - - /// Slow path for a whole SIMD vector or vector-pair access: describe the - /// full instruction in the CPU request fields (`vector_dest`, optionally - /// `second_*`) and trap out; the machine completes every byte and resumes - /// after the instruction. - fn simd_trap_access( - &self, - address: Value, - bytes: u64, - rt: u8, - rt2: Option, - post: Option<(u8, Value)>, - pc: u64, - trap: Trap, - ) { - self.store(offset_of!(Cpu, address), address); - self.byte(offset_of!(Cpu, width), bytes); - self.byte(offset_of!(Cpu, dest), rt as u64); - self.byte(offset_of!(Cpu, vector_dest), 1); - if let Some(rt2) = rt2 { - let second = self.imm(self.i64, B::Add, address, if bytes == 16 { 16 } else { 8 }); - self.store(offset_of!(Cpu, second_address), second); - self.byte(offset_of!(Cpu, second_width), bytes); - self.byte(offset_of!(Cpu, second_dest), rt2 as u64); - self.byte(offset_of!(Cpu, second_vector), 1); - self.byte(offset_of!(Cpu, second_pending), 1); - } - if let Some((rn, value)) = post { - self.store(offset_of!(Cpu, writeback_value), value); - self.byte(offset_of!(Cpu, writeback_dest), rn as u64); - self.byte(offset_of!(Cpu, writeback), 1); - } - self.trap(pc.wrapping_add(4), trap); - } -} -fn arithmetic_op(op: ArithOp) -> B { - if op == ArithOp::Sub { B::Sub } else { B::Add } -} -fn logic_op(op: LogicOp) -> B { - match op { - LogicOp::BitAnd => B::BitAnd, - LogicOp::BitOr => B::BitOr, - LogicOp::BitXor => B::BitXor, - } -} -fn low_mask(n: u8) -> u64 { - if n >= 64 { u64::MAX } else { (1u64 << n) - 1 } -} - pub fn compile(guest_pc: u64, bytes: &[u8]) -> Result { compile_with(guest_pc, bytes, false) } @@ -639,20 +90,21 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< if bytes.len() % 4 != 0 { return Err(Error::UnsupportedInstruction); } - let l = Lower::new(); + let mut lifter = Lifter::new(CpuEnvironment::new(inline_memory)); let mut pc = guest_pc; let mut ended = false; for chunk in bytes.chunks_exact(4) { - l.pending.set(l.pending.get() + 1); let word = u32::from_le_bytes(chunk.try_into().unwrap()); - let instruction = match dispatch(word) { - Ok(Dispatch::Guest(insn)) => insn, + match dispatch(word) { + Ok(Dispatch::Guest(instruction)) => { + ended = instruction.terminates_with(inline_memory); + lifter.lift(pc, instruction)?; + } Ok(Dispatch::Host(host_word)) => { // The host runs this word (see `host::run`), and it ends the block. - l.store(offset_of!(Cpu, value), l.k(l.i64, host_word as u64)); - l.trap(pc.wrapping_add(4), Trap::Host); + let (builder, env) = lifter.parts(); + env.host(builder, pc, host_word); ended = true; - break; } Err(_) => { eprintln!("compile decode failure at pc {pc:x}: word {word:08x}, full block:"); @@ -667,1479 +119,6 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } return Err(Error::Decode { word, pc }); } - }; - ended = instruction.terminates_with(inline_memory); - match instruction { - Instruction::Movz(a) | Instruction::Movn(a) => { - let t = l.ty(a.width); - let bits = (a.immediate as u64) << a.shift; - let value = if matches!(instruction, Instruction::Movn(_)) { - !bits - } else { - bits - }; - if a.rd != 31 { - l.put(a.rd, l.k(t, value)); - } - } - Instruction::Movk(a) => { - if a.rd != 31 { - let t = l.ty(a.width); - let old = l.reg(t, a.rd, false); - let kept = l.imm(t, B::BitAnd, old, !(0xffffu64 << a.shift)); - l.put( - a.rd, - l.imm(t, B::BitOr, kept, (a.immediate as u64) << a.shift), - ); - } - } - Instruction::ArithImm(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, false); - let rhs = l.k(t, a.immediate as u64); - let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); - if a.flags { - l.put(a.rd, r); - l.store( - offset_of!(Cpu, flags), - l.flags(a.width, a.op, lhs, rhs, r, None), - ); - } else { - l.put_sp(a.rd, r) - } - } - Instruction::ArithReg(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let rhs = l.shift(a.width, l.reg(t, a.rm, true), a.shift, a.amount as u64); - let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); - if a.rd != 31 { - l.put(a.rd, r) - } - if a.flags { - l.store( - offset_of!(Cpu, flags), - l.flags(a.width, a.op, lhs, rhs, r, None), - ); - } - } - Instruction::ArithExt(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, false); - let extended = l.extend(l.reg(l.i64, a.rm, true), a.extend); - let rhs = l.cv(t, l.imm(l.i64, B::Shl, extended, a.amount as u64)); - let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); - if a.flags { - l.put(a.rd, r); - l.store( - offset_of!(Cpu, flags), - l.flags(a.width, a.op, lhs, rhs, r, None), - ); - } else { - l.put_sp(a.rd, r) - } - } - Instruction::LogicReg(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let shifted = l.shift(a.width, l.reg(t, a.rm, true), a.shift, a.amount as u64); - let rhs = if a.invert { - l.imm(t, B::BitXor, shifted, u64::MAX) - } else { - shifted - }; - let r = l.bin(t, logic_op(a.op), lhs, rhs); - l.put(a.rd, r); - if a.flags { - l.logic_flags(a.width, r) - } - } - Instruction::LogicImm(a) => { - let t = l.ty(a.width); - let r = l.imm(t, logic_op(a.op), l.reg(t, a.rn, true), a.immediate); - if a.flags { - l.put(a.rd, r); - l.logic_flags(a.width, r) - } else { - l.put_sp(a.rd, r) - } - } - Instruction::ArithCarry(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let loaded = l.reg(t, a.rm, true); - let rhs = if a.op == ArithOp::Sub { - l.imm(t, B::BitXor, loaded, u64::MAX) - } else { - loaded - }; - let carry = l.cv( - t, - l.imm( - l.i32, - B::BitAnd, - l.imm(l.i32, B::Shr, l.load(l.i32, offset_of!(Cpu, flags)), 29), - 1, - ), - ); - let partial = l.bin(t, B::Add, lhs, rhs); - let r = l.bin(t, B::Add, partial, carry); - if a.rd != 31 { - l.put(a.rd, r) - } - if a.flags { - let out = l.bin( - l.boolean, - B::BitOr, - l.cmp(C::Ult, partial, lhs), - l.cmp(C::Ult, r, partial), - ); - l.store( - offset_of!(Cpu, flags), - l.flags(a.width, ArithOp::Add, lhs, rhs, r, Some(out)), - ); - } - } - Instruction::CondCompare(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let rhs = match a.operand { - CompareOperand::Register(r) => l.reg(t, r, true), - CompareOperand::Immediate(n) => l.k(t, n as u64), - }; - let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); - let flags = l.flags(a.width, a.op, lhs, rhs, r, None); - l.store( - offset_of!(Cpu, flags), - l.sel( - l.i32, - l.condition(a.cond), - flags, - l.k(l.i32, (a.nzcv as u64) << 28), - ), - ); - } - Instruction::Variable(a) => lower_variable(&l, a), - Instruction::Bitfield(a) => lower_bitfield(&l, a), - Instruction::Unary(a) => lower_unary(&l, a), - Instruction::Extract(a) => { - let t = l.ty(a.width); - let low = l.reg(t, a.rm, true); - let r = if a.lsb == 0 { - low - } else { - l.bin( - t, - B::BitOr, - l.imm(t, B::Shr, low, a.lsb as u64), - l.imm( - t, - B::Shl, - l.reg(t, a.rn, true), - a.width as u64 - a.lsb as u64, - ), - ) - }; - if a.rd != 31 { - l.put(a.rd, r) - } - } - Instruction::Mul(a) => { - let t = l.ty(a.width); - let p = l.bin(t, B::Mul, l.reg(t, a.rn, true), l.reg(t, a.rm, true)); - let r = l.bin(t, arithmetic_op(a.op), l.reg(t, a.ra, true), p); - if a.rd != 31 { - l.put(a.rd, r) - } - } - Instruction::MulLong(a) => { - let e = if a.signed { Extend::Sxtw } else { Extend::Uxtw }; - let p = l.bin( - l.i64, - B::Mul, - l.extend(l.reg(l.i64, a.rn, true), e), - l.extend(l.reg(l.i64, a.rm, true), e), - ); - let r = l.bin(l.i64, arithmetic_op(a.op), l.reg(l.i64, a.ra, true), p); - if a.rd != 31 { - l.put(a.rd, r) - } - } - Instruction::MulHigh(a) => lower_high(&l, a), - Instruction::Adr(a) => { - let here = if a.page { pc & 0xfffffffffffff000 } else { pc }; - l.put(a.rd, l.k(l.i64, here.wrapping_add(a.offset as u64))); - } - Instruction::Csel(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let rhs = l.reg(t, a.rm, true); - let alt = match a.opc { - 0 => rhs, - 1 => l.imm(t, B::Add, rhs, 1), - 2 => l.imm(t, B::BitXor, rhs, u64::MAX), - 3 => l.bin(t, B::Sub, l.k(t, 0), rhs), - _ => unreachable!(), - }; - if a.rd != 31 { - l.put(a.rd, l.sel(t, l.condition(a.cond), lhs, alt)); - } - } - Instruction::TestBranch(a) => { - let t = l.ty(a.width); - let v = l.reg(t, a.rt, true); - let subject = if let Some(bit) = a.bit_index { - l.imm(t, B::BitAnd, v, 1u64 << bit) - } else { - v - }; - l.branch( - pc, - a.offset, - l.cmp(if a.negate { C::Ne } else { C::Eq }, subject, l.k(t, 0)), - ); - } - Instruction::BCond(a) => l.branch(pc, a.offset, l.condition(a.cond)), - Instruction::B(offset) => l.ret(l.k(l.i64, pc.wrapping_add(offset as u64))), - Instruction::Call(a) => { - if a.link { - l.put(30, l.k(l.i64, pc.wrapping_add(4))) - } - l.ret(l.k(l.i64, pc.wrapping_add(a.target as u64))); - } - Instruction::Indirect(a) => { - let target = l.reg(l.i64, a.rn, false); - if a.link { - l.put(30, l.k(l.i64, pc.wrapping_add(4))) - } - l.ret(target); - } - Instruction::System(a) => lower_system(&l, a), - Instruction::Memory(a) => { - let trap = if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }; - if inline_memory { - let (address, post) = l.address_split(a.rn, a.addressing); - l.memory_access(address, a.size, a.op, a.rt, a.signed, post, |l| { - setup_memory(l, address, a.size, a.rt, a.signed, a.op); - if let Some((rn, value)) = post { - l.store(offset_of!(Cpu, writeback_value), value); - l.byte(offset_of!(Cpu, writeback_dest), rn as u64); - l.byte(offset_of!(Cpu, writeback), 1); - } - l.trap(pc.wrapping_add(4), trap); - }); - } else { - let address = l.address(a.rn, a.addressing); - setup_memory(&l, address, a.size, a.rt, a.signed, a.op); - l.trap(pc.wrapping_add(4), trap); - } - } - Instruction::Literal(a) => { - let address = l.k(l.i64, pc.wrapping_add(a.offset as u64)); - if inline_memory { - l.memory_access( - address, - a.size, - MemoryOp::Load, - a.rt, - SignExtend::None, - None, - |l| { - setup_memory( - l, - address, - a.size, - a.rt, - SignExtend::None, - MemoryOp::Load, - ); - l.trap(pc.wrapping_add(4), Trap::Load); - }, - ); - } else { - setup_memory(&l, address, a.size, a.rt, SignExtend::None, MemoryOp::Load); - l.trap(pc.wrapping_add(4), Trap::Load); - } - } - Instruction::SimdLiteral(a) => { - let address = l.k(l.i64, pc.wrapping_add(a.offset as u64)); - l.simd_trap_access(address, a.bytes as u64, a.rt, None, None, pc, Trap::Load); - } - Instruction::Exclusive(a) => { - setup_memory( - &l, - l.reg(l.i64, a.rn, false), - a.size, - a.rt, - SignExtend::None, - a.op, - ); - if a.op == MemoryOp::Store { - l.byte(offset_of!(Cpu, status_dest), a.rs as u64) - } - l.byte( - offset_of!(Cpu, exclusive), - if a.op == MemoryOp::Store { - CpuExclusive::Store as u64 - } else { - CpuExclusive::Load as u64 - }, - ); - l.trap( - pc.wrapping_add(4), - if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }, - ); - } - Instruction::Atomic(a) => { - // Read-modify-write needs a host atomic, so it always takes the machine - // slow path. `dests` receive the words read (31 discards) and `operands` - // are what the machine combines with memory (see `AtomicKind::apply`). - let (dests, operands): ([u8; 2], &[u8]) = match a.kind { - AtomicKind::Casp => ([a.rs, a.rs + 1], &[a.rs, a.rs + 1, a.rt, a.rt + 1]), - AtomicKind::Cas => ([a.rs, 31], &[a.rs, a.rt]), - AtomicKind::LoadPair => ([a.rt, a.rt2], &[]), - AtomicKind::StorePair => ([31, 31], &[a.rt, a.rt2]), - _ => ([a.rt, 31], &[a.rs]), - }; - l.store(offset_of!(Cpu, address), l.reg(l.i64, a.rn, false)); - l.byte(offset_of!(Cpu, width), a.size as u64); - l.byte(offset_of!(Cpu, dest), dests[0] as u64); - l.byte(offset_of!(Cpu, atomic_kind), a.kind as u64); - for (slot, &dest) in dests.iter().enumerate() { - l.byte(offset_of!(Cpu, atomic_dests) + slot, dest as u64); - } - for (slot, ®) in operands.iter().enumerate() { - // Register numbers past 30 read as XZR. - l.store( - offset_of!(Cpu, atomic_operands) + slot * 8, - l.reg(l.i64, reg, true), - ); - } - if a.kind == AtomicKind::StorePair { - l.byte(offset_of!(Cpu, status_dest), a.rs as u64); - } - l.trap(pc.wrapping_add(4), Trap::Atomic); - } - Instruction::SimdStruct(a) => { - use volt_target::aarch64::simd_struct::StructDesc; - // Element-wise accesses with lane merging need the machine slow path. - let d = a.desc; - let address = l.reg(l.i64, a.rn, false); - l.store(offset_of!(Cpu, address), address); - for (field, value) in [ - (offset_of!(StructDesc, store), u64::from(d.store)), - (offset_of!(StructDesc, shape), d.shape as u64), - (offset_of!(StructDesc, registers), u64::from(d.registers)), - ( - offset_of!(StructDesc, interleaved), - u64::from(d.interleaved), - ), - (offset_of!(StructDesc, esize), u64::from(d.esize)), - (offset_of!(StructDesc, lane), u64::from(d.lane)), - (offset_of!(StructDesc, q), u64::from(d.q)), - (offset_of!(StructDesc, rt), u64::from(d.rt)), - ] { - l.byte(offset_of!(Cpu, structure) + field, value); - } - let step = match a.post { - StructPost::None => None, - StructPost::Immediate => Some(l.k(l.i64, d.bytes())), - StructPost::Register(rm) => Some(l.reg(l.i64, rm, true)), - }; - if let Some(step) = step { - l.store( - offset_of!(Cpu, writeback_value), - l.bin(l.i64, B::Add, address, step), - ); - l.byte(offset_of!(Cpu, writeback_dest), a.rn as u64); - l.byte(offset_of!(Cpu, writeback), 1); - } - l.trap(pc.wrapping_add(4), Trap::SimdStruct); - } - Instruction::AddressTranslate(a) => { - l.store(offset_of!(Cpu, address), l.reg(l.i64, a.rt, true)); - l.store( - offset_of!(Cpu, value), - l.k(l.i64, u64::from(a.write) | u64::from(a.user) << 1), - ); - l.trap(pc.wrapping_add(4), Trap::AddressTranslate); - } - Instruction::Pair(a) => { - let trap = if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }; - if inline_memory { - let (address, post) = l.address_split(a.rn, a.addressing); - l.pair_access(address, a, post, |l| { - setup_pair(l, address, a); - if let Some((rn, value)) = post { - l.store(offset_of!(Cpu, writeback_value), value); - l.byte(offset_of!(Cpu, writeback_dest), rn as u64); - l.byte(offset_of!(Cpu, writeback), 1); - } - l.trap(pc.wrapping_add(4), trap); - }); - } else { - let address = l.address(a.rn, a.addressing); - setup_pair(&l, address, a); - l.trap(pc.wrapping_add(4), trap); - } - } - Instruction::Clrex(_) => l.byte(offset_of!(Cpu, monitor_valid), 0), - Instruction::Sev | Instruction::Sevl => l.byte(offset_of!(Cpu, event_set), 1), - Instruction::SimdImm(a) => { - let at = offset_of!(Cpu, v) + a.rd as usize * 16; - match a.combine { - ImmCombine::Set => { - l.store(at, l.k(l.i64, a.low)); - l.store(at + 8, l.k(l.i64, a.high)); - } - ImmCombine::Or | ImmCombine::AndNot => { - // A 64-bit form (`q == false`) also clears the upper half. - let (op, imm) = match a.combine { - ImmCombine::Or => (B::BitOr, a.low), - _ => (B::BitAnd, !a.low), - }; - l.store(at, l.imm(l.i64, op, l.load(l.i64, at), imm)); - l.store( - at + 8, - if a.q { - l.imm(l.i64, op, l.load(l.i64, at + 8), imm) - } else { - l.k(l.i64, 0) - }, - ); - } - } - } - Instruction::SimdMove { rd, rn } => { - let at_d = offset_of!(Cpu, v) + rd as usize * 16; - let at_n = offset_of!(Cpu, v) + rn as usize * 16; - let lo = l.load(l.i64, at_n); - let hi = l.load(l.i64, at_n + 8); - l.store(at_d, lo); - l.store(at_d + 8, hi); - } - Instruction::SimdInsert(a) => { - // Replace one lane of the destination with the low bits of a GPR. - let ebits = (1u64 << a.esize) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let per_dw = (64 / ebits) as usize; - let word = a.index as usize / per_dw; - let shift = (a.index as usize % per_dw) as u64 * ebits; - let old = l.load(l.i64, at_d + word * 8); - let cleared = l.bin(l.i64, B::BitAnd, old, l.k(l.i64, !(emask << shift))); - let value = l.bin( - l.i64, - B::BitAnd, - l.reg(l.i64, a.rn, true), - l.k(l.i64, emask), - ); - l.store( - at_d + word * 8, - l.bin(l.i64, B::BitOr, cleared, l.imm(l.i64, B::Shl, value, shift)), - ); - } - Instruction::SimdInsertElement(a) => { - // Copy one lane of Vn over one lane of Vd. The source is read before - // the store, so `rd == rn` is fine. - let ebits = (1u64 << a.esize) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let per_dw = (64 / ebits) as usize; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let (src_word, src_shift) = ( - a.src as usize / per_dw, - (a.src as usize % per_dw) as u64 * ebits, - ); - let (dst_word, dst_shift) = ( - a.dst as usize / per_dw, - (a.dst as usize % per_dw) as u64 * ebits, - ); - let source = l.load(l.i64, at_n + src_word * 8); - let value = l.imm( - l.i64, - B::BitAnd, - l.imm(l.i64, B::Shr, source, src_shift), - emask, - ); - let old = l.load(l.i64, at_d + dst_word * 8); - let cleared = l.imm(l.i64, B::BitAnd, old, !(emask << dst_shift)); - l.store( - at_d + dst_word * 8, - l.bin( - l.i64, - B::BitOr, - cleared, - l.imm(l.i64, B::Shl, value, dst_shift), - ), - ); - } - Instruction::SimdExtract(a) => { - // Move one lane into the low bits of a GPR, zero-extended. - let ebits = (1u64 << a.esize) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let per_dw = (64 / ebits) as usize; - let word = a.index as usize / per_dw; - let shift = (a.index as usize % per_dw) as u64 * ebits; - let source = l.load(l.i64, at_n + word * 8); - let value = l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, source, shift), emask); - l.put(a.rd, value); - } - Instruction::SimdDupElement(a) => { - let ebits = (1u64 << a.esize) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let vn_lo = l.load(l.i64, at_n); - let vn_hi = l.load(l.i64, at_n + 8); - // Locate the requested element across the 128-bit source. - let lane = a.index as usize; - let lanes_per_dw = (64 / ebits) as usize; - let src = if lane / lanes_per_dw == 0 { - vn_lo - } else { - vn_hi - }; - let shift = (lane % lanes_per_dw) as u64 * ebits; - let element = l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, src, shift), emask); - // Replicate the element across the destination lanes. - let mut acc = l.k(l.i64, 0); - let lanes = (64 / ebits) as usize; - for lane in 0..lanes { - acc = l.bin( - l.i64, - B::BitOr, - acc, - l.imm(l.i64, B::Shl, element, lane as u64 * ebits), - ); - } - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - l.store(at_d, acc); - l.store(at_d + 8, if a.q { acc } else { l.k(l.i64, 0) }); - } - Instruction::SimdDup { rd, rn, esize, q } => { - let r = l.reg(l.i64, rn, true); - let val64 = match esize { - 0 => { - let b = l.imm(l.i64, B::BitAnd, r, 0xff); - l.bin(l.i64, B::Mul, b, l.k(l.i64, 0x0101_0101_0101_0101)) - } - 1 => { - let h = l.imm(l.i64, B::BitAnd, r, 0xffff); - l.bin(l.i64, B::Mul, h, l.k(l.i64, 0x0001_0001_0001_0001)) - } - 2 => { - let w = l.imm(l.i64, B::BitAnd, r, 0xffff_ffff); - l.bin(l.i64, B::BitOr, w, l.imm(l.i64, B::Shl, w, 32)) - } - _ => r, - }; - let at = offset_of!(Cpu, v) + rd as usize * 16; - l.store(at, val64); - let hi = if q { val64 } else { l.k(l.i64, 0) }; - l.store(at + 8, hi); - } - Instruction::SimdExt(a) => { - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let vn_lo = l.load(l.i64, at_n); - let vn_hi = l.load(l.i64, at_n + 8); - let vm_lo = l.load(l.i64, at_m); - let vm_hi = l.load(l.i64, at_m + 8); - let (lo, hi) = if a.imm == 0 { - (vn_lo, vn_hi) - } else if a.imm == 8 { - (vn_hi, vm_lo) - } else if a.imm < 8 { - let s = a.imm as u64 * 8; - let r = (8 - a.imm as u64) * 8; - let lo = l.bin( - l.i64, - B::BitOr, - l.imm(l.i64, B::Shr, vn_lo, s), - l.imm(l.i64, B::Shl, vn_hi, r), - ); - let hi = l.bin( - l.i64, - B::BitOr, - l.imm(l.i64, B::Shr, vn_hi, s), - l.imm(l.i64, B::Shl, vm_lo, r), - ); - (lo, hi) - } else { - let shift = a.imm as u64 - 8; - let s = shift * 8; - let r = (8 - shift) * 8; - let lo = l.bin( - l.i64, - B::BitOr, - l.imm(l.i64, B::Shr, vn_hi, s), - l.imm(l.i64, B::Shl, vm_lo, r), - ); - let hi = l.bin( - l.i64, - B::BitOr, - l.imm(l.i64, B::Shr, vm_lo, s), - l.imm(l.i64, B::Shl, vm_hi, r), - ); - (lo, hi) - }; - l.store(at_d, lo); - l.store(at_d + 8, hi); - } - Instruction::SimdCompareZero(a) => { - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.i64, at_n); - let vn_hi = l.load(l.i64, at_n + 8); - let ebits = (1u64 << a.size) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let halves = if a.q { 2 } else { 1 }; - for half in 0..2 { - if half >= halves { - l.store(at_d + half * 8, l.k(l.i64, 0)); - continue; - } - let v = if half == 0 { vn_lo } else { vn_hi }; - let mut acc = l.k(l.i64, 0); - let lanes = 8 >> a.size; - for lane in 0..lanes { - let shift = lane as u64 * ebits; - let field = l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, v, shift), emask); - let is_zero = l.cmp(C::Eq, field, l.k(l.i64, 0)); - let bits = l.sel(l.i64, is_zero, l.k(l.i64, emask), l.k(l.i64, 0)); - acc = l.bin(l.i64, B::BitOr, acc, l.imm(l.i64, B::Shl, bits, shift)); - } - l.store(at_d + half * 8, acc); - } - } - Instruction::SimdTable(a) => { - // Table bytes live in `len + 1` consecutive registers. Each index - // byte selects a table byte; out-of-range indices leave zero for - // `tbl` and keep the destination byte for `tbx`. - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let vm_lo = l.load(l.i64, at_m); - let vm_hi = l.load(l.i64, at_m + 8); - let regs = a.len as usize + 1; - let mut table = Vec::with_capacity(regs * 2); - for i in 0..regs { - let reg = (a.rn as usize + i) % 32; - let at = offset_of!(Cpu, v) + reg * 16; - table.push(l.load(l.i64, at)); - table.push(l.load(l.i64, at + 8)); - } - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let old_lo = if a.extend { - Some(l.load(l.i64, at_d)) - } else { - None - }; - let old_hi = if a.extend { - Some(l.load(l.i64, at_d + 8)) - } else { - None - }; - let halves = if a.q { 2 } else { 1 }; - let limit = (16 * regs) as u64; - for half in 0..2 { - if half >= halves { - l.store(at_d + half * 8, l.k(l.i64, 0)); - continue; - } - let indices = if half == 0 { vm_lo } else { vm_hi }; - let old = if half == 0 { old_lo } else { old_hi }; - let mut acc = l.k(l.i64, 0); - for byte in 0..8u64 { - let shift = byte * 8; - let index = - l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, indices, shift), 0xff); - let in_range = l.cmp(C::Ult, index, l.k(l.i64, limit)); - let mut value = l.k(l.i64, 0); - let sel = |c: Value, x: Value, y: Value| l.sel(l.i64, c, x, y); - for (b, entry) in table.iter().enumerate() { - let here = l.cmp( - C::Eq, - l.bin(l.i64, B::Shr, index, l.k(l.i64, 3)), - l.k(l.i64, b as u64), - ); - let byte_at = l.imm( - l.i64, - B::BitAnd, - l.bin( - l.i64, - B::Shr, - *entry, - l.bin( - l.i64, - B::Shl, - l.imm(l.i64, B::BitAnd, index, 7), - l.k(l.i64, 3), - ), - ), - 0xff, - ); - value = - sel(l.bin(l.boolean, B::BitAnd, in_range, here), byte_at, value); - } - if a.extend { - value = sel( - in_range, - value, - l.imm( - l.i64, - B::BitAnd, - l.imm(l.i64, B::Shr, old.unwrap(), shift), - 0xff, - ), - ); - } - acc = l.bin(l.i64, B::BitOr, acc, l.imm(l.i64, B::Shl, value, shift)); - } - l.store(at_d + half * 8, acc); - } - } - Instruction::SimdFmov(a) => { - // Pure bit move; no FP arithmetic involved. - if a.to_fp { - let v = l.reg(l.i64, a.rn, true); - let bits = if a.double { - v - } else { - l.imm(l.i64, B::BitAnd, v, 0xffff_ffff) - }; - let at = offset_of!(Cpu, v) + a.rd as usize * 16; - l.store(at, bits); - l.store(at + 8, l.k(l.i64, 0)); - } else { - let at = offset_of!(Cpu, v) + a.rn as usize * 16; - let v = l.load(l.i64, at); - let bits = if a.double { - v - } else { - l.imm(l.i64, B::BitAnd, v, 0xffff_ffff) - }; - l.put(a.rd, bits); - } - } - Instruction::SimdAlu(a) => { - use volt_target::aarch64::decode::SimdAluOp; - // Bitwise logical operations work on whole halves, lane width free. - if matches!( - a.op, - SimdAluOp::AddV | SimdAluOp::UMinV | SimdAluOp::UMaxV | SimdAluOp::UAddLV - ) { - // Horizontal add, minimum, maximum or widening add: reduce every - // lane of the source and store the result in lane 0, zeroing the - // rest. `UAddLV` keeps twice the lane width. - let ebits = (1u64 << a.size) * 8; - let mask_of = |bits: u64| { - if bits >= 64 { - u64::MAX - } else { - (1u64 << bits) - 1 - } - }; - let emask = mask_of(ebits); - let result_mask = if a.op == SimdAluOp::UAddLV { - mask_of(ebits * 2) - } else { - emask - }; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.i64, at_n); - let vn_hi = l.load(l.i64, at_n + 8); - let halves = if a.q { 2 } else { 1 }; - let mut total = if a.op == SimdAluOp::UMinV { - l.k(l.i64, emask) - } else { - l.k(l.i64, 0) - }; - for half in 0..halves { - let v = if half == 0 { vn_lo } else { vn_hi }; - for lane in 0..(8 >> a.size) { - let shift = lane as u64 * ebits; - let field = - l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, v, shift), emask); - total = match a.op { - SimdAluOp::UMinV => { - l.sel(l.i64, l.cmp(C::Ult, field, total), field, total) - } - SimdAluOp::UMaxV => { - l.sel(l.i64, l.cmp(C::Ugt, field, total), field, total) - } - _ => l.bin(l.i64, B::Add, total, field), - }; - } - } - l.store( - at_d, - l.bin(l.i64, B::BitAnd, total, l.k(l.i64, result_mask)), - ); - l.store(at_d + 8, l.k(l.i64, 0)); - } else if matches!( - a.op, - SimdAluOp::And - | SimdAluOp::Bic - | SimdAluOp::Orr - | SimdAluOp::Orn - | SimdAluOp::Eor - | SimdAluOp::Bsl - | SimdAluOp::Bit - | SimdAluOp::Bif - ) { - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - for half in 0..2 { - if half == 1 && !a.q { - l.store(at_d + 8, l.k(l.i64, 0)); - continue; - } - let x = l.load(l.i64, at_n + half * 8); - let y = l.load(l.i64, at_m + half * 8); - let not = |v| l.bin(l.i64, B::BitXor, v, l.k(l.i64, u64::MAX)); - let r = match a.op { - SimdAluOp::And => l.bin(l.i64, B::BitAnd, x, y), - SimdAluOp::Bic => l.bin(l.i64, B::BitAnd, x, not(y)), - SimdAluOp::Orr => l.bin(l.i64, B::BitOr, x, y), - SimdAluOp::Orn => l.bin(l.i64, B::BitOr, x, not(y)), - SimdAluOp::Eor => l.bin(l.i64, B::BitXor, x, y), - // The destination is both a source mask and the - // merge base for BSL/BIT/BIF. - _ => { - let d = l.load(l.i64, at_d + half * 8); - match a.op { - SimdAluOp::Bsl => l.bin( - l.i64, - B::BitOr, - l.bin(l.i64, B::BitAnd, x, d), - l.bin(l.i64, B::BitAnd, y, not(d)), - ), - _ => { - let selected = match a.op { - SimdAluOp::Bit => l.bin( - l.i64, - B::BitAnd, - l.bin(l.i64, B::BitXor, d, x), - y, - ), - _ => l.bin( - l.i64, - B::BitAnd, - l.bin(l.i64, B::BitXor, d, x), - not(y), - ), - }; - l.bin(l.i64, B::BitXor, d, selected) - } - } - } - }; - l.store(at_d + half * 8, r); - } - } else { - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.i64, at_n); - let vn_hi = l.load(l.i64, at_n + 8); - let vm_lo = l.load(l.i64, at_m); - let vm_hi = l.load(l.i64, at_m + 8); - let need_d = matches!(a.op, SimdAluOp::Mla | SimdAluOp::Mls); - // Pairwise forms consume adjacent lanes of the full source pair: - // lane 0/1 of Vn and Vm, then lane 2/3, and so on. - if matches!( - a.op, - SimdAluOp::UMaxP - | SimdAluOp::SMaxP - | SimdAluOp::UMinP - | SimdAluOp::SMinP - | SimdAluOp::AddP - ) { - let ebits = (1u64 << a.size) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let ssel = |c: Value, x: Value, y: Value| l.sel(l.i64, c, x, y); - let sources = [ - l.load(l.i64, at_n), - l.load(l.i64, at_n + 8), - l.load(l.i64, at_m), - l.load(l.i64, at_m + 8), - ]; - // Each source is a 128-bit vector. Lane `2k` pairs with lane - // `2k + 1` and the result lands in lane `k`; the reduction of Vn - // fills the destination's low half and Vm its high half. - let per_source = (128 / ebits) as usize; - let reduced_lanes = per_source / 2; - let mut out = [l.k(l.i64, 0); 2]; - for (which, half) in [(0usize, 0usize), (2usize, 1usize)] { - let mut acc = l.k(l.i64, 0); - for k in 0..reduced_lanes { - let lanes_per_dw = (64 / ebits) as usize; - let read = |lane: usize| { - let double = lane / lanes_per_dw; - let shift = (lane % lanes_per_dw) as u64 * ebits; - l.imm( - l.i64, - B::BitAnd, - l.imm(l.i64, B::Shr, sources[which + double], shift), - emask, - ) - }; - let lo = read(2 * k); - let hi = read(2 * k + 1); - let r = match a.op { - SimdAluOp::AddP => l.bin(l.i64, B::Add, lo, hi), - SimdAluOp::UMaxP => ssel(l.cmp(C::Ugt, lo, hi), lo, hi), - SimdAluOp::UMinP => ssel(l.cmp(C::Ult, lo, hi), lo, hi), - SimdAluOp::SMaxP => ssel(l.cmp(C::Sgt, lo, hi), lo, hi), - _ => ssel(l.cmp(C::Slt, lo, hi), lo, hi), - }; - let masked = l.bin(l.i64, B::BitAnd, r, l.k(l.i64, emask)); - // per_source lanes reduce to exactly 64 bits, so the - // reduced lanes always pack into one doubleword. - acc = l.bin( - l.i64, - B::BitOr, - acc, - l.imm(l.i64, B::Shl, masked, k as u64 * ebits), - ); - } - out[half] = acc; - } - l.store(at_d, out[0]); - l.store(at_d + 8, if a.q { out[1] } else { l.k(l.i64, 0) }); - } else { - let vd_lo = if need_d { - Some(l.load(l.i64, at_d)) - } else { - None - }; - let vd_hi = if need_d { - Some(l.load(l.i64, at_d + 8)) - } else { - None - }; - let ebits = (1u64 << a.size) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let smax = u64::MAX >> (65 - ebits); - let smin = !smax; - let halves = if a.q { 2 } else { 1 }; - for half in 0..2 { - if half >= halves { - l.store(at_d + half * 8, l.k(l.i64, 0)); - continue; - } - let vn = if half == 0 { vn_lo } else { vn_hi }; - let vm = if half == 0 { vm_lo } else { vm_hi }; - let vd = if half == 0 { vd_lo } else { vd_hi }; - let mut acc = l.k(l.i64, 0); - let lanes = 8 >> a.size; - for lane in 0..lanes { - let shift = lane as u64 * ebits; - let fld = |v| { - l.imm(l.i64, B::BitAnd, l.imm(l.i64, B::Shr, v, shift), emask) - }; - let ext = |v: Value| { - let up = l.bin(l.i64, B::Shl, v, l.k(l.i64, 64 - ebits)); - l.imm(l.i64, B::Sar, up, 64 - ebits) - }; - let an = fld(vn); - let bn = fld(vm); - let sa = ext(an); - let sb = ext(bn); - let dn = vd.map(|v| fld(v)); - let ssel = |c: Value, x: Value, y: Value| l.sel(l.i64, c, x, y); - let ssat = |v: Value| { - let over = l.cmp(C::Sgt, v, l.k(l.i64, smax)); - let under = l.cmp(C::Slt, v, l.k(l.i64, smin)); - ssel(over, l.k(l.i64, smax), ssel(under, l.k(l.i64, smin), v)) - }; - let r = match a.op { - SimdAluOp::Add => l.bin(l.i64, B::Add, an, bn), - SimdAluOp::Sub => l.bin(l.i64, B::Sub, an, bn), - SimdAluOp::Mul => l.bin(l.i64, B::Mul, an, bn), - SimdAluOp::Mla => l.bin( - l.i64, - B::Add, - dn.unwrap(), - l.bin(l.i64, B::Mul, an, bn), - ), - SimdAluOp::Mls => l.bin( - l.i64, - B::Sub, - dn.unwrap(), - l.bin(l.i64, B::Mul, an, bn), - ), - SimdAluOp::CmEq => { - ssel(l.cmp(C::Eq, an, bn), l.k(l.i64, emask), l.k(l.i64, 0)) - } - SimdAluOp::CmGt => ssel( - l.cmp(C::Sgt, sa, sb), - l.k(l.i64, emask), - l.k(l.i64, 0), - ), - SimdAluOp::CmGe => ssel( - l.cmp(C::Sge, sa, sb), - l.k(l.i64, emask), - l.k(l.i64, 0), - ), - SimdAluOp::CmHi => ssel( - l.cmp(C::Ugt, an, bn), - l.k(l.i64, emask), - l.k(l.i64, 0), - ), - SimdAluOp::CmHs => ssel( - l.cmp(C::Uge, an, bn), - l.k(l.i64, emask), - l.k(l.i64, 0), - ), - SimdAluOp::CmTst => ssel( - l.cmp( - C::Ne, - l.bin(l.i64, B::BitAnd, an, bn), - l.k(l.i64, 0), - ), - l.k(l.i64, emask), - l.k(l.i64, 0), - ), - SimdAluOp::UMin => ssel(l.cmp(C::Ult, an, bn), an, bn), - SimdAluOp::UMax => ssel(l.cmp(C::Ugt, an, bn), an, bn), - SimdAluOp::SMin => ssel(l.cmp(C::Slt, sa, sb), an, bn), - SimdAluOp::SMax => ssel(l.cmp(C::Sgt, sa, sb), an, bn), - SimdAluOp::UAbd => { - let d = l.bin(l.i64, B::Sub, an, bn); - ssel( - l.cmp(C::Uge, an, bn), - d, - l.bin(l.i64, B::Sub, l.k(l.i64, 0), d), - ) - } - SimdAluOp::SAbd => { - let d = l.bin(l.i64, B::Sub, sa, sb); - let neg = l.bin(l.i64, B::Sub, l.k(l.i64, 0), d); - ssel(l.cmp(C::Sge, sa, sb), d, neg) - } - SimdAluOp::SQAdd => ssat(l.bin(l.i64, B::Add, sa, sb)), - SimdAluOp::UQAdd => { - let s = l.bin(l.i64, B::Add, an, bn); - let carry = l.cmp(C::Ult, s, an); - let big = l.cmp(C::Ugt, s, l.k(l.i64, emask)); - ssel( - l.bin(l.i64, B::BitOr, carry, big), - l.k(l.i64, emask), - s, - ) - } - SimdAluOp::SQSub => ssat(l.bin(l.i64, B::Sub, sa, sb)), - SimdAluOp::UQSub => { - let borrow = l.cmp(C::Ugt, bn, an); - ssel(borrow, l.k(l.i64, 0), l.bin(l.i64, B::Sub, an, bn)) - } - SimdAluOp::SHAdd - | SimdAluOp::UHAdd - | SimdAluOp::SHSub - | SimdAluOp::UHSub - | SimdAluOp::SRHAdd - | SimdAluOp::URHAdd => { - let signed_op = matches!( - a.op, - SimdAluOp::SHAdd | SimdAluOp::SHSub | SimdAluOp::SRHAdd - ); - // Signed forms operate on sign-extended lanes, which - // keeps the halving shift meaningful for negatives. - let (op_a, op_b) = - if signed_op { (sa, sb) } else { (an, bn) }; - let shr = |v: Value| { - if signed_op { - l.imm(l.i64, B::Sar, v, 1) - } else { - l.imm(l.i64, B::Shr, v, 1) - } - }; - let k = l.bin(l.i64, B::Sub, shr(op_a), shr(op_b)); - let k = if matches!( - a.op, - SimdAluOp::SHAdd - | SimdAluOp::UHAdd - | SimdAluOp::SRHAdd - | SimdAluOp::URHAdd - ) { - l.bin(l.i64, B::Add, shr(op_a), shr(op_b)) - } else { - k - }; - let a0 = l.bin(l.i64, B::BitAnd, op_a, l.k(l.i64, 1)); - let b0 = l.bin(l.i64, B::BitAnd, op_b, l.k(l.i64, 1)); - let corr = match a.op { - SimdAluOp::SHAdd | SimdAluOp::UHAdd => { - l.bin(l.i64, B::BitAnd, a0, b0) - } - SimdAluOp::SRHAdd | SimdAluOp::URHAdd => { - l.bin(l.i64, B::BitOr, a0, b0) - } - _ => l.bin( - l.i64, - B::BitAnd, - l.bin(l.i64, B::BitXor, a0, l.k(l.i64, 1)), - b0, - ), - }; - if matches!( - a.op, - SimdAluOp::SHAdd - | SimdAluOp::UHAdd - | SimdAluOp::SRHAdd - | SimdAluOp::URHAdd - ) { - l.bin(l.i64, B::Add, k, corr) - } else { - l.bin(l.i64, B::Sub, k, corr) - } - } - SimdAluOp::SSHl - | SimdAluOp::USHl - | SimdAluOp::SQShl - | SimdAluOp::UQShl - | SimdAluOp::SQRShl - | SimdAluOp::UQRShl => { - let signed_op = matches!( - a.op, - SimdAluOp::SSHl | SimdAluOp::SQShl | SimdAluOp::SQRShl - ); - let rounding = - matches!(a.op, SimdAluOp::SQRShl | SimdAluOp::UQRShl); - let saturating = matches!( - a.op, - SimdAluOp::SQShl - | SimdAluOp::UQShl - | SimdAluOp::SQRShl - | SimdAluOp::UQRShl - ); - // Shift amount: the lane's low byte, sign-extended to - // 64 bits (negative means a right shift). - let amt = l.bin( - l.i64, - B::Shl, - l.bin(l.i64, B::BitAnd, bn, l.k(l.i64, 0xff)), - l.k(l.i64, 56), - ); - let amt = l.bin(l.i64, B::Sar, amt, l.k(l.i64, 56)); - let left = l.cmp(C::Sge, amt, l.k(l.i64, 0)); - let n = ssel( - left, - amt, - l.bin(l.i64, B::Sub, l.k(l.i64, 0), amt), - ); - let shl = |v: Value, k: Value| l.bin(l.i64, B::Shl, v, k); - let sar = |v: Value, k: Value| l.bin(l.i64, B::Sar, v, k); - let shr = |v: Value, k: Value| l.bin(l.i64, B::Shr, v, k); - // Beyond the lane width: a plain right shift keeps the - // sign fill (or zero), a rounding shift collapses to zero, - // and a saturating left shift pins to the lane bound. - let sign_fill = if signed_op { - ssel( - l.cmp(C::Slt, sa, l.k(l.i64, 0)), - l.k(l.i64, emask), - l.k(l.i64, 0), - ) - } else { - l.k(l.i64, 0) - }; - let sat_fill = if signed_op { - ssel( - l.cmp(C::Slt, sa, l.k(l.i64, 0)), - l.k(l.i64, smin), - l.k(l.i64, smax), - ) - } else { - l.k(l.i64, emask) - }; - let far = ssel( - left, - if saturating { sat_fill } else { l.k(l.i64, 0) }, - if rounding { l.k(l.i64, 0) } else { sign_fill }, - ); - // Shift when the amount stays inside the lane width. - // Clamping to `ebits - 1` keeps the host shift defined. - let kc = ssel( - l.cmp(C::Uge, n, l.k(l.i64, ebits)), - l.k(l.i64, ebits - 1), - n, - ); - let rnd = if rounding { - shl( - l.k(l.i64, 1), - l.bin(l.i64, B::Sub, kc, l.k(l.i64, 1)), - ) - } else { - l.k(l.i64, 0) - }; - let opnd = if signed_op { sa } else { an }; - let t = if rounding { - // The rounding constant must not leak out of the lane. - l.bin( - l.i64, - B::BitAnd, - l.bin(l.i64, B::Add, opnd, rnd), - l.k(l.i64, emask), - ) - } else { - opnd - }; - let right = if rounding { - if signed_op { sar(t, kc) } else { shr(t, kc) } - } else if signed_op { - sar(opnd, kc) - } else { - shr(opnd, kc) - }; - let shifted = shl(opnd, kc); - let moved = ssel(left, shifted, right); - let moved = - ssel(l.cmp(C::Uge, n, l.k(l.i64, ebits)), far, moved); - if !saturating { - moved - } else if signed_op { - // Saturation only matters for left shifts; the - // right-shift result already fits the lane. Below - // the 64-bit lane the shifted value is exact, so the - // lane bounds test it directly; a 64-bit lane needs - // the round-trip check because the shift wraps. - let ov = if ebits == 64 { - l.cmp(C::Ne, sar(shifted, kc), opnd) - } else { - l.bin( - l.boolean, - B::BitOr, - l.cmp(C::Sgt, shifted, l.k(l.i64, smax)), - l.cmp(C::Slt, shifted, l.k(l.i64, smin)), - ) - }; - let sat = ssel( - l.cmp(C::Slt, sa, l.k(l.i64, 0)), - l.k(l.i64, smin), - l.k(l.i64, smax), - ); - ssel(left, ssel(ov, sat, moved), moved) - } else { - let ov = if ebits == 64 { - l.cmp(C::Ne, shr(shifted, kc), an) - } else { - l.cmp(C::Ugt, shifted, l.k(l.i64, emask)) - }; - ssel(left, ssel(ov, l.k(l.i64, emask), moved), moved) - } - } - _ => l.k(l.i64, 0), - }; - let masked = l.bin(l.i64, B::BitAnd, r, l.k(l.i64, emask)); - acc = l.bin( - l.i64, - B::BitOr, - acc, - l.imm(l.i64, B::Shl, masked, shift), - ); - } - l.store(at_d + half * 8, acc); - } - } - } - } - Instruction::SimdPair(a) => { - let trap = if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }; - let (address, post) = l.address_split(a.rn, a.addressing); - if !inline_memory { - l.simd_trap_access(address, a.bytes as u64, a.rt, Some(a.rt2), post, pc, trap); - } else { - let at_rt = offset_of!(Cpu, v) + a.rt as usize * 16; - let at_rt2 = offset_of!(Cpu, v) + a.rt2 as usize * 16; - let slow_block = l.new_block(); - let store = a.op == MemoryOp::Store; - let (bytes, chunk_count) = if a.bytes == 16 { (16, 4) } else { (8, 2) }; - let starts: [Value; 4] = [ - address, - l.imm(l.i64, B::Add, address, 8), - l.imm(l.i64, B::Add, address, 16), - l.imm(l.i64, B::Add, address, 24), - ]; - // stores: (reg_offset_base, half) per chunk - let src: [(usize, u64); 4] = if bytes == 16 { - [(at_rt, 0), (at_rt + 8, 0), (at_rt2, 0), (at_rt2 + 8, 0)] - } else { - [(at_rt, 0), (at_rt2, 0), (at_rt, 0), (at_rt, 0)] - }; - let mut hosts: [Option; 4] = [None; 4]; - for i in 0..chunk_count { - let next = l.new_block(); - let (hit, host) = l.dtlb_probe(starts[i], Size::Double, store); - l.branch_if(hit, next, slow_block); - l.block.set(next); - hosts[i] = Some(host); - } - if store { - for i in 0..chunk_count { - let host = hosts[i].unwrap(); - l.store_ptr(host, l.load(l.i64, src[i].0)); - } - } else { - for i in 0..chunk_count { - let host = hosts[i].unwrap(); - l.store(src[i].0, l.load_ptr(l.i64, host)); - } - // zero the untransferred half of each 128-bit register - if bytes == 8 { - l.store(at_rt + 8, l.k(l.i64, 0)); - l.store(at_rt2 + 8, l.k(l.i64, 0)); - } - } - if let Some((rn, value)) = post { - l.put_sp(rn, value); - } - // slow path, built last so every fast edge into it exists. - let resume = l.block.get(); - l.block.set(slow_block); - l.simd_trap_access(address, bytes as u64, a.rt, Some(a.rt2), post, pc, trap); - l.block.set(resume); - } - } - Instruction::SimdMemory(a) => { - let trap = if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }; - let (address, post) = l.address_split(a.rn, a.addressing); - if !inline_memory { - l.simd_trap_access(address, a.bytes as u64, a.rt, None, post, pc, trap); - } else { - let at_rt = offset_of!(Cpu, v) + a.rt as usize * 16; - let slow_block = l.new_block(); - let store = a.op == MemoryOp::Store; - // One chunk for widths up to 8 bytes, two 8-byte chunks for 16. - let wide = a.bytes == 16; - let size = match a.bytes { - 1 => Size::Byte, - 2 => Size::Half, - 4 => Size::Word, - _ => Size::Double, - }; - let chunk_count = if wide { 2 } else { 1 }; - let starts: [Value; 2] = [address, l.imm(l.i64, B::Add, address, 8)]; - let chunk_ty = l.int((a.bytes.min(8) as u16) * 8); - let src: [usize; 2] = [at_rt, at_rt + 8]; - let mut hosts: [Option; 2] = [None; 2]; - for i in 0..chunk_count { - let next = l.new_block(); - let (hit, host) = l.dtlb_probe(starts[i], size, store); - l.branch_if(hit, next, slow_block); - l.block.set(next); - hosts[i] = Some(host); - } - if store { - for i in 0..chunk_count { - let host = hosts[i].unwrap(); - l.store_ptr(host, l.cv(chunk_ty, l.load(l.i64, src[i]))); - } - } else { - for i in 0..chunk_count { - let host = hosts[i].unwrap(); - l.store(src[i], l.cv(l.i64, l.load_ptr(chunk_ty, host))); - } - // Scalar loads zero the whole upper register, not just the - // untransferred half. - if a.bytes <= 8 { - l.store(at_rt + 8, l.k(l.i64, 0)); - if a.bytes < 8 { - let kept = (1u64 << (a.bytes * 8)) - 1; - let lo = l.imm(l.i64, B::BitAnd, l.load(l.i64, at_rt), kept); - l.store(at_rt, lo); - } - } - } - if let Some((rn, value)) = post { - l.put_sp(rn, value); - } - let resume = l.block.get(); - l.block.set(slow_block); - l.simd_trap_access(address, a.bytes as u64, a.rt, None, post, pc, trap); - l.block.set(resume); - } - } - Instruction::Nop - | Instruction::Hint(_) - | Instruction::Prefetch(_) - | Instruction::PrefetchLiteral(_) - | Instruction::CacheOp(_) - | Instruction::Dsb(_) - | Instruction::Dmb(_) - | Instruction::Yield => {} - Instruction::Wfe => l.trap(pc.wrapping_add(4), Trap::Wfe), - Instruction::Brk(imm) => { - l.store(offset_of!(Cpu, address), l.k(l.i64, u64::from(imm))); - l.trap(pc, Trap::Brk) - } - Instruction::Isb(_) => l.trap(pc.wrapping_add(4), Trap::Isb), - Instruction::Tlbi(_) => l.trap(pc.wrapping_add(4), Trap::Tlbi), - Instruction::DcZva(rt) => { - l.store(offset_of!(Cpu, address), l.reg(l.i64, rt, true)); - l.trap(pc.wrapping_add(4), Trap::DcZva) - } - Instruction::Eret => l.trap(pc.wrapping_add(4), Trap::Eret), - Instruction::Svc(imm) => { - l.store(offset_of!(Cpu, address), l.k(l.i64, u64::from(imm))); - l.trap(pc.wrapping_add(4), Trap::Svc) - } - Instruction::Psci => l.trap(pc.wrapping_add(4), Trap::Psci), - Instruction::Wfi => l.trap(pc.wrapping_add(4), Trap::Wfi), } pc = pc.wrapping_add(4); if ended { @@ -2147,644 +126,10 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } } if !ended { - l.ret(l.k(l.i64, pc)) - } - Ok(Block::new(native::compile_owned(l.f.into_inner())?)) -} -/// Describe a GPR pair access to the machine: the first access in the usual -/// request fields, the second as `second_*`, completed after the first. -fn setup_pair(l: &Lower, address: Value, a: Pair) { - setup_memory(l, address, a.size, a.rt, a.signed, a.op); - l.store( - offset_of!(Cpu, second_address), - l.imm(l.i64, B::Add, address, a.size as u64), - ); - l.byte(offset_of!(Cpu, second_width), a.size as u64); - l.byte(offset_of!(Cpu, second_dest), a.rt2 as u64); - if a.signed != SignExtend::None { - l.byte(offset_of!(Cpu, second_signed), a.signed as u64) - } - if a.op == MemoryOp::Store { - l.store(offset_of!(Cpu, second_value), l.reg(l.i64, a.rt2, true)) - } - l.byte(offset_of!(Cpu, second_pending), 1); -} -fn setup_memory(l: &Lower, address: Value, size: Size, rt: u8, signed: SignExtend, op: MemoryOp) { - l.store(offset_of!(Cpu, address), address); - l.byte(offset_of!(Cpu, width), size as u64); - l.byte(offset_of!(Cpu, dest), rt as u64); - if signed != SignExtend::None { - l.byte(offset_of!(Cpu, load_signed), signed as u64) - } - if op == MemoryOp::Store { - l.store(offset_of!(Cpu, value), l.reg(l.i64, rt, true)) - } -} - -fn lower_variable(l: &Lower, a: Variable) { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let rhs = l.reg(t, a.rm, true); - let bits = a.width as u64; - let r = match a.op { - VariableOp::Lsl | VariableOp::Lsr | VariableOp::Asr | VariableOp::Ror => { - let distance = l.imm(t, B::BitAnd, rhs, bits - 1); - match a.op { - VariableOp::Lsl => l.bin(t, B::Shl, lhs, distance), - VariableOp::Lsr => l.bin(t, B::Shr, lhs, distance), - VariableOp::Asr => l.bin(t, B::Sar, lhs, distance), - VariableOp::Ror => { - let back = l.imm( - t, - B::BitAnd, - l.bin(t, B::Sub, l.k(t, bits), distance), - bits - 1, - ); - l.bin( - t, - B::BitOr, - l.bin(t, B::Shr, lhs, distance), - l.bin(t, B::Shl, lhs, back), - ) - } - _ => unreachable!(), - } - } - VariableOp::Udiv | VariableOp::Sdiv => { - let zero = l.k(t, 0); - let one = l.k(t, 1); - let by_zero = l.cmp(C::Eq, rhs, zero); - if a.op == VariableOp::Udiv { - let safe = l.sel(t, by_zero, one, rhs); - l.sel(t, by_zero, zero, l.bin(t, B::UDiv, lhs, safe)) - } else { - let minus_one = l.cmp(C::Eq, rhs, l.k(t, u64::MAX)); - let safe = l.sel(t, minus_one, one, l.sel(t, by_zero, one, rhs)); - let quotient = l.bin(t, B::SDiv, lhs, safe); - let negated = l.bin(t, B::Sub, zero, lhs); - l.sel(t, by_zero, zero, l.sel(t, minus_one, negated, quotient)) - } - } - }; - if a.rd != 31 { - l.put(a.rd, r) - } -} -fn lower_bitfield(l: &Lower, a: Bitfield) { - let t = l.ty(a.width); - let bits = a.width as u64; - let len = a.len as u64; - let lsb = a.lsb as u64; - let source = l.reg(t, a.rn, true); - let mask = low_mask(a.len); - let width_mask = if a.width == Width::X64 { - u64::MAX - } else { - u32::MAX as u64 - }; - let extract = |signed| { - let raised = l.imm(t, B::Shl, source, bits - lsb - len); - l.shift( - a.width, - raised, - if signed { Shift::Asr } else { Shift::Lsr }, - bits - len, - ) - }; - let r = match a.op { - BitfieldOp::Unsigned => { - if a.extract { - extract(false) - } else { - l.imm( - t, - B::Shl, - l.imm(t, B::BitAnd, source, mask & width_mask), - lsb, - ) - } - } - BitfieldOp::Signed => { - if a.extract { - extract(true) - } else { - let raised = l.imm(t, B::Shl, source, bits - len); - let extended = l.shift(a.width, raised, Shift::Asr, bits - len); - l.imm(t, B::Shl, extended, lsb) - } - } - BitfieldOp::Insert => { - let value = if a.extract { - extract(false) - } else { - l.imm( - t, - B::Shl, - l.imm(t, B::BitAnd, source, mask & width_mask), - lsb, - ) - }; - let place = if a.extract { 0 } else { lsb }; - let hole = (mask << place) & width_mask; - let kept = l.imm(t, B::BitAnd, l.reg(t, a.rd, true), !hole & width_mask); - l.bin(t, B::BitOr, kept, value) - } - }; - if a.rd != 31 { - l.put(a.rd, r) - } -} -fn repeated(pattern: u64, unit: u32, bits: u32) -> u64 { - let mut r = 0; - let mut at = 0; - while at < bits { - r |= pattern << at; - at += unit - } - r -} -fn swap(l: &Lower, t: Type, v: Value, mask: u64, n: u64) -> Value { - let low = l.imm(t, B::BitAnd, v, mask); - let high = l.imm(t, B::BitAnd, l.imm(t, B::Shr, v, n), mask); - l.bin(t, B::BitOr, l.imm(t, B::Shl, low, n), high) -} -fn lower_unary(l: &Lower, a: Unary) { - let t = l.ty(a.width); - let bits = a.width as u32; - let mut v = l.reg(t, a.rn, true); - match a.op { - UnaryOp::Rev16 => v = swap(l, t, v, repeated(0xff, 16, bits), 8), - UnaryOp::Rev32 => { - v = swap(l, t, v, repeated(0xff, 16, bits), 8); - v = swap(l, t, v, repeated(0xffff, 32, bits), 16); - } - UnaryOp::Rev | UnaryOp::Rbit => { - if a.op == UnaryOp::Rbit { - for (mask, unit, n) in [(1, 2, 1), (3, 4, 2), (15, 8, 4)] { - v = swap(l, t, v, repeated(mask, unit, bits), n) - } - } - v = swap(l, t, v, repeated(0xff, 16, bits), 8); - v = swap(l, t, v, repeated(0xffff, 32, bits), 16); - if bits == 64 { - v = l.bin( - t, - B::BitOr, - l.imm(t, B::Shl, v, 32), - l.imm(t, B::Shr, v, 32), - ) - } - } - UnaryOp::Clz => { - let original = v; - let mut count = l.k(t, 0); - let mut step = bits / 2; - while step >= 1 { - let top = l.imm(t, B::Shr, v, (bits - step) as u64); - let empty = l.cmp(C::Eq, top, l.k(t, 0)); - v = l.sel(t, empty, l.imm(t, B::Shl, v, step as u64), v); - count = l.sel(t, empty, l.imm(t, B::Add, count, step as u64), count); - step /= 2; - } - v = l.sel( - t, - l.cmp(C::Eq, original, l.k(t, 0)), - l.k(t, bits as u64), - count, - ); - } - } - if a.rd != 31 { - l.put(a.rd, v) - } -} -fn lower_high(l: &Lower, a: MultiplyHigh) { - let t = l.i64; - let x = l.reg(t, a.rn, true); - let y = l.reg(t, a.rm, true); - let xl = l.imm(t, B::BitAnd, x, 0xffffffff); - let yl = l.imm(t, B::BitAnd, y, 0xffffffff); - let xh = l.imm(t, B::Shr, x, 32); - let yh = l.imm(t, B::Shr, y, 32); - let ll = l.bin(t, B::Mul, xl, yl); - let lh = l.bin(t, B::Mul, xl, yh); - let hl = l.bin(t, B::Mul, xh, yl); - let hh = l.bin(t, B::Mul, xh, yh); - let middle = l.bin( - t, - B::Add, - l.bin( - t, - B::Add, - l.imm(t, B::Shr, ll, 32), - l.imm(t, B::BitAnd, lh, 0xffffffff), - ), - l.imm(t, B::BitAnd, hl, 0xffffffff), - ); - let mut high = l.bin( - t, - B::Add, - l.bin(t, B::Add, hh, l.imm(t, B::Shr, lh, 32)), - l.bin( - t, - B::Add, - l.imm(t, B::Shr, hl, 32), - l.imm(t, B::Shr, middle, 32), - ), - ); - if a.signed { - let zero = l.k(t, 0); - let xn = l.bin(t, B::Sub, zero, l.imm(t, B::Shr, x, 63)); - let yn = l.bin(t, B::Sub, zero, l.imm(t, B::Shr, y, 63)); - high = l.bin(t, B::Sub, high, l.bin(t, B::BitAnd, y, xn)); - high = l.bin(t, B::Sub, high, l.bin(t, B::BitAnd, x, yn)); - } - if a.rd != 31 { - l.put(a.rd, high) - } -} - -fn system_offset(r: SystemRegister) -> Option { - use SystemRegister::*; - Some(match r { - SpsrEl1 => offset_of!(CpuSystem, spsr_el1), - ElrEl1 => offset_of!(CpuSystem, elr_el1), - EsrEl1 => offset_of!(CpuSystem, esr_el1), - FarEl1 => offset_of!(CpuSystem, far_el1), - ParEl1 => offset_of!(CpuSystem, par_el1), - IsrEl1 => offset_of!(CpuSystem, isr_el1), - IccSreEl1 => offset_of!(CpuSystem, icc_sre_el1), - IccPmrEl1 => offset_of!(CpuSystem, icc_pmr_el1), - IccBpr0El1 => offset_of!(CpuSystem, icc_bpr0_el1), - IccCtlrEl1 => offset_of!(CpuSystem, icc_ctlr_el1), - IccIgrpen0El1 => offset_of!(CpuSystem, icc_igrpen0_el1), - IccIgrpen1El1 => offset_of!(CpuSystem, icc_igrpen1_el1), - IccBpr1El1 => offset_of!(CpuSystem, icc_bpr1_el1), - CntpCtlEl0 => offset_of!(CpuSystem, cntp_ctl_el0), - CntpCvalEl0 => offset_of!(CpuSystem, cntp_cval_el0), - ActlrEl1 => offset_of!(CpuSystem, actlr_el1), - VbarEl1 => offset_of!(CpuSystem, vbar_el1), - SctlrEl2 => offset_of!(CpuSystem, sctlr_el2), - TcrEl2 => offset_of!(CpuSystem, tcr_el2), - Ttbr0El2 => offset_of!(CpuSystem, ttbr0_el2), - Ttbr1El2 => offset_of!(CpuSystem, ttbr1_el2), - MairEl2 => offset_of!(CpuSystem, mair_el2), - VbarEl2 => offset_of!(CpuSystem, vbar_el2), - ElrEl2 => offset_of!(CpuSystem, elr_el2), - SpsrEl2 => offset_of!(CpuSystem, spsr_el2), - EsrEl2 => offset_of!(CpuSystem, esr_el2), - FarEl2 => offset_of!(CpuSystem, far_el2), - HcrEl2 => offset_of!(CpuSystem, hcr_el2), - TpidrEl2 => offset_of!(CpuSystem, tpidr_el2), - CntvoffEl2 => offset_of!(CpuSystem, cntvoff_el2), - CpacrEl1 => offset_of!(CpuSystem, cpacr_el1), - SctlrEl1 => offset_of!(CpuSystem, sctlr_el1), - Ttbr0El1 => offset_of!(CpuSystem, ttbr0_el1), - Ttbr1El1 => offset_of!(CpuSystem, ttbr1_el1), - TcrEl1 => offset_of!(CpuSystem, tcr_el1), - MairEl1 => offset_of!(CpuSystem, mair_el1), - TpidrEl0 => offset_of!(CpuSystem, tpidr_el0), - TpidrroEl0 => offset_of!(CpuSystem, tpidrro_el0), - CntvctEl0 => offset_of!(CpuSystem, cntvct_el0), - CntvCtlEl0 => offset_of!(CpuSystem, cntv_ctl_el0), - CntvCvalEl0 => offset_of!(CpuSystem, cntv_cval_el0), - TpidrEl1 => offset_of!(CpuSystem, tpidr_el1), - ContextidrEl1 => offset_of!(CpuSystem, contextidr_el1), - CsselrEl1 => offset_of!(CpuSystem, csselr_el1), - MdscrEl1 => offset_of!(CpuSystem, mdscr_el1), - CntkctlEl1 => offset_of!(CpuSystem, cntkctl_el1), - OslarEl1 => offset_of!(CpuSystem, oslar_el1), - OsdlrEl1 => offset_of!(CpuSystem, osdlr_el1), - Fpcr => offset_of!(CpuSystem, fpcr), - Fpsr => offset_of!(CpuSystem, fpsr), - Daif => offset_of!(CpuSystem, daif), - Apple(slot) => offset_of!(CpuSystem, apple) + 8 * usize::from(slot), - // After `system` in `Cpu`: relative to it, like every other register here. - AppleBank(slot) => { - offset_of!(Cpu, apple_bank) - offset_of!(Cpu, system) + 8 * usize::from(slot) - } - MidrEl1 => offset_of!(CpuSystem, midr_el1), - IdAa64pfr0El1 => offset_of!(CpuSystem, id_aa64pfr0_el1), - IdAa64mmfr0El1 => offset_of!(CpuSystem, id_aa64mmfr0_el1), - PacKey(slot) => offset_of!(CpuSystem, pac_keys) + 8 * usize::from(slot), - IdAa64isar1El1 => offset_of!(CpuSystem, id_aa64isar1_el1), - _ => return None, - }) -} - -/// EL1 registers that `HCR_EL2.E2H` redirects to their EL2 banks when accessed from EL2: -/// (EL1 field offset, EL2 field offset). -fn vhe_banks(r: SystemRegister) -> Option<(usize, usize)> { - use SystemRegister::*; - Some(match r { - SctlrEl1 => ( - offset_of!(CpuSystem, sctlr_el1), - offset_of!(CpuSystem, sctlr_el2), - ), - Ttbr0El1 => ( - offset_of!(CpuSystem, ttbr0_el1), - offset_of!(CpuSystem, ttbr0_el2), - ), - Ttbr1El1 => ( - offset_of!(CpuSystem, ttbr1_el1), - offset_of!(CpuSystem, ttbr1_el2), - ), - TcrEl1 => ( - offset_of!(CpuSystem, tcr_el1), - offset_of!(CpuSystem, tcr_el2), - ), - MairEl1 => ( - offset_of!(CpuSystem, mair_el1), - offset_of!(CpuSystem, mair_el2), - ), - VbarEl1 => ( - offset_of!(CpuSystem, vbar_el1), - offset_of!(CpuSystem, vbar_el2), - ), - ElrEl1 => ( - offset_of!(CpuSystem, elr_el1), - offset_of!(CpuSystem, elr_el2), - ), - SpsrEl1 => ( - offset_of!(CpuSystem, spsr_el1), - offset_of!(CpuSystem, spsr_el2), - ), - EsrEl1 => ( - offset_of!(CpuSystem, esr_el1), - offset_of!(CpuSystem, esr_el2), - ), - FarEl1 => ( - offset_of!(CpuSystem, far_el1), - offset_of!(CpuSystem, far_el2), - ), - _ => return None, - }) -} -fn lower_system(l: &Lower, a: System) { - use SystemRegister::*; - l.flush_counter(); - let at = offset_of!(Cpu, system); - let level = match a.register { - SpEl0 => Some(0u64), - SpEl1 => Some(1), - SpEl2 => Some(2), - _ => None, - }; - if let Some(level) = level { - let current = l.cv(l.i64, l.load(l.i8, at + offset_of!(CpuSystem, el))); - let at_level = l.cmp(C::Eq, current, l.k(l.i64, level)); - let live = if level <= 1 { - let el1 = l.cmp(C::Eq, current, l.k(l.i64, 1)); - let selected = l.cmp( - if level == 0 { C::Eq } else { C::Ne }, - l.load(l.i8, at + offset_of!(CpuSystem, spsel)), - l.k(l.i8, 0), - ); - let el1_selected = l.bin(l.boolean, B::BitAnd, el1, selected); - if level == 0 { - l.bin(l.boolean, B::BitOr, at_level, el1_selected) - } else { - el1_selected - } - } else { - at_level - }; - let p = l.sel( - l.ptr, - live, - l.addr(Lower::reg_at(31)), - l.addr(at + offset_of!(CpuSystem, sp_el) + level as usize * 8), - ); - if a.read { - l.put(a.rt, l.load_ptr(l.i64, p)) - } else { - l.store_ptr(p, l.reg(l.i64, a.rt, true)) - } - return; - } - if let Some((el1, el2)) = vhe_banks(a.register) { - // HCR_EL2.E2H: EL1 register names accessed from EL2 reach the EL2 banks. - let current = l.cv(l.i64, l.load(l.i8, at + offset_of!(CpuSystem, el))); - let el2_mode = l.cmp(C::Eq, current, l.k(l.i64, 2)); - let e2h = l.cmp( - C::Ne, - l.imm( - l.i64, - B::BitAnd, - l.load(l.i64, at + offset_of!(CpuSystem, hcr_el2)), - super::cpu::HCR_E2H, - ), - l.k(l.i64, 0), - ); - let redirect = l.bin(l.boolean, B::BitAnd, el2_mode, e2h); - let p = l.sel(l.ptr, redirect, l.addr(at + el2), l.addr(at + el1)); - if a.read { - l.put(a.rt, l.load_ptr(l.i64, p)); - } else { - l.store_ptr(p, l.reg(l.i64, a.rt, true)); - } - return; - } - match a.register { - CurrentEl => l.put( - a.rt, - l.imm( - l.i64, - B::Shl, - l.cv(l.i64, l.load(l.i8, at + offset_of!(CpuSystem, el))), - 2, - ), - ), - Pan | PanSet => { - // PSTATE.PAN lives in bit 22 of the register form. - let pan = at + offset_of!(CpuSystem, pan); - if a.read { - l.put(a.rt, l.imm(l.i64, B::Shl, l.load(l.i64, pan), 22)); - } else if a.register == PanSet { - l.store(pan, l.k(l.i64, u64::from(a.immediate & 1))); - } else { - let bit = l.imm(l.i64, B::Shr, l.reg(l.i64, a.rt, true), 22); - l.store(pan, l.imm(l.i64, B::BitAnd, bit, 1)); - } - } - RazWi | AppleZero => { - if a.read { - l.put(a.rt, l.k(l.i64, 0)); - } - } - SpSel | SpSelSet => { - let selected = l.load(l.i8, at + offset_of!(CpuSystem, spsel)); - if a.read { - l.put(a.rt, l.cv(l.i64, selected)); - return; - } - let new = if a.register == SpSelSet { - l.k(l.i8, u64::from(a.immediate & 1)) - } else { - l.cv(l.i8, l.imm(l.i64, B::BitAnd, l.reg(l.i64, a.rt, true), 1)) - }; - // The live stack pointer belongs to bank `SP_EL1` when SPSel is 1 - // and `SP_EL0` when it is 0. Park it in its bank, then load the - // other one. Selecting the bank already in use is a store and a - // reload of the same slot. - let banks = at + offset_of!(CpuSystem, sp_el); - let bank = |select: Value| { - l.sel( - l.ptr, - l.cmp(C::Ne, select, l.k(l.i8, 0)), - l.addr(banks + 8), - l.addr(banks), - ) - }; - let live = l.load(l.i64, Lower::reg_at(31)); - l.store_ptr(bank(selected), live); - let incoming = l.load_ptr(l.i64, bank(new)); - l.store(Lower::reg_at(31), incoming); - l.store(at + offset_of!(CpuSystem, spsel), new); - } - DaifSet | DaifClear => { - let p = at + offset_of!(CpuSystem, daif); - let before = l.load(l.i64, p); - let updated = l.imm( - l.i64, - if a.register == DaifSet { - B::BitOr - } else { - B::BitAnd - }, - before, - if a.register == DaifSet { - a.immediate as u64 - } else { - (a.immediate as u64) ^ 0xf - }, - ); - l.store(p, updated); - } - Nzcv => { - if a.read { - l.put(a.rt, l.cv(l.i64, l.load(l.i32, offset_of!(Cpu, flags)))) - } else { - l.store( - offset_of!(Cpu, flags), - l.imm( - l.i32, - B::BitAnd, - l.cv(l.i32, l.reg(l.i64, a.rt, true)), - 0xf0000000, - ), - ) - } - } - OslsrEl1 => { - let either = l.bin( - l.i64, - B::BitOr, - l.load(l.i64, at + offset_of!(CpuSystem, oslar_el1)), - l.load(l.i64, at + offset_of!(CpuSystem, osdlr_el1)), - ); - let locked = l.cmp(C::Ne, either, l.k(l.i64, 0)); - l.put(a.rt, l.sel(l.i64, locked, l.k(l.i64, 2), l.k(l.i64, 0))); - } - CcsidrEl1 => { - // CSSELR bit 0 is InD, bits 3:1 the level - 1; L2 has no separate I-side. - use super::identification::{CCSIDR_L1D, CCSIDR_L1I, CCSIDR_L2}; - let select = l.imm( - l.i64, - B::BitAnd, - l.load(l.i64, at + offset_of!(CpuSystem, csselr_el1)), - 0xf, - ); - let is = |n: u64| l.cmp(C::Eq, select, l.k(l.i64, n)); - let l1_instruction = l.sel(l.i64, is(1), l.k(l.i64, CCSIDR_L1I), l.k(l.i64, CCSIDR_L2)); - l.put( - a.rt, - l.sel(l.i64, is(0), l.k(l.i64, CCSIDR_L1D), l1_instruction), - ); - } - CntvTvalEl0 | CntpTvalEl0 => { - // The timer value is the signed 32-bit distance to the compare value: - // reads give the low 32 bits of `CVAL - count`, writes `CVAL = count + sext(value)`. - // The physical timer shares the virtual count, as CNTVOFF_EL2 is zero without EL2. - let counter = l.load(l.i64, at + offset_of!(CpuSystem, cntvct_el0)); - let compare = at - + if a.register == CntpTvalEl0 { - offset_of!(CpuSystem, cntp_cval_el0) - } else { - offset_of!(CpuSystem, cntv_cval_el0) - }; - if a.read { - let distance = l.bin(l.i64, B::Sub, l.load(l.i64, compare), counter); - l.put(a.rt, l.imm(l.i64, B::BitAnd, distance, 0xffff_ffff)); - } else { - let delta = l.extend(l.reg(l.i64, a.rt, true), Extend::Sxtw); - l.store(compare, l.bin(l.i64, B::Add, counter, delta)); - } - } - CntpctEl0 => { - // Writes are ignored: the counter is not writable from software. - if a.read { - l.put(a.rt, l.load(l.i64, at + offset_of!(CpuSystem, cntvct_el0))); - } - } - IccIar1El1 => { - // No group 1 interrupt is delivered by this model, so acknowledge reads spurious. - if a.read { - l.put(a.rt, l.k(l.i64, 1023)); - } - } - IccEoir1El1 => {} - Daif => { - let p = at + offset_of!(CpuSystem, daif); - if a.read { - l.put(a.rt, l.imm(l.i64, B::Shl, l.load(l.i64, p), 6)) - } else { - l.store( - p, - l.imm( - l.i64, - B::BitAnd, - l.imm(l.i64, B::Shr, l.reg(l.i64, a.rt, true), 6), - 0xf, - ), - ); - } - } - _ => { - if let Some(constant) = super::identification::value(a.register) { - l.put(a.rt, l.k(l.i64, constant)); - return; - } - let offset = system_offset(a.register).expect("decoded system register has storage"); - let p = at + offset; - if a.read { - let v = if a.register == CntvCtlEl0 { - let counter = l.load(l.i64, at + offset_of!(CpuSystem, cntvct_el0)); - let compare = l.load(l.i64, at + offset_of!(CpuSystem, cntv_cval_el0)); - let reached = l.cmp(C::Uge, counter, compare); - l.bin( - l.i64, - B::BitOr, - l.load(l.i64, p), - l.sel(l.i64, reached, l.k(l.i64, 4), l.k(l.i64, 0)), - ) - } else { - l.load(l.i64, p) - }; - l.put(a.rt, v); - } else { - let from = l.reg(l.i64, a.rt, true); - l.store( - p, - if a.register == CntvCtlEl0 { - l.imm(l.i64, B::BitAnd, from, 3) - } else { - from - }, - ); - } - } + lifter.exit(pc); } + let (function, _) = lifter.into_parts(); + Ok(Block::new(native::compile_owned(function)?)) } #[cfg(test)] @@ -2808,183 +153,6 @@ mod tests { assert_eq!(cpu.flags, 0x80000000); } #[test] - fn zero_division_and_signed_overflow_do_not_trap_host() { - // UDIV X2,X0,X1; SDIV X3,X0,X1. - let block = compile(0, &bytes(&[0x9ac10802, 0x9ac10c03])).unwrap(); - let mut cpu = Cpu::default(); - cpu.x[0] = 1 << 63; - cpu.x[1] = 0; - block.run(&mut cpu); - assert_eq!(cpu.x[2], 0); - assert_eq!(cpu.x[3], 0); - cpu.x[1] = u64::MAX; - block.run(&mut cpu); - assert_eq!(cpu.x[3], 1 << 63); - } - #[test] - fn signed_and_unsigned_integer_ops_select_distinct_semantics() { - // sdiv/udiv/asrv (w and x), smulh/umulh, sxtw, sbfx, asr #imm, then - // `cmp` + `cset` for each signed/unsigned condition on w and x. - let words = [ - 0x1ac10c02, 0x1ac10803, 0x1ac12804, 0x9ac12805, 0x9b417c06, 0x9bc17c07, 0x93407c08, - 0x93442c09, 0x13037c0a, 0x6b01001f, 0x1a9fa7eb, 0x1a9fb7ec, 0x1a9f27ed, 0x1a9f97ee, - 0x1a9fd7ef, 0x1a9fc7f0, 0x1a9f87f1, 0xeb01001f, 0x9a9fa7f2, 0x9a9f27f3, 0x9a9fd7f4, - ]; - let block = compile(0, &bytes(&words)).unwrap(); - let pairs: [(u64, u64); 9] = [ - (0xffff_ffff_8000_0005, 3), - (0x8000_0000, 0xffff_ffff), - (0x7fff_ffff_ffff_ffff, 0x8000_0000_0000_0000), - (0x8000_0000_0000_0000, u64::MAX), - (3, 0), - (7, 7), - (0xffff_ffff_0000_0007, 0x0000_0001_0000_0009), - (0x8000_0001, 2), - (2, 0x8000_0001), - ]; - for (a, b) in pairs { - let mut cpu = Cpu::default(); - cpu.x[0] = a; - cpu.x[1] = b; - block.run(&mut cpu); - let (w, wb) = (a as u32, b as u32); - let (sw, swb) = (w as i32, wb as i32); - let sdiv = if wb == 0 { - 0 - } else { - sw.wrapping_div(swb) as u32 - }; - let udiv = if wb == 0 { 0 } else { w / wb }; - assert_eq!(cpu.x[2], sdiv as u64, "sdiv w {a:#x} {b:#x}"); - assert_eq!(cpu.x[3], udiv as u64, "udiv w {a:#x} {b:#x}"); - assert_eq!(cpu.x[4], (sw >> (wb & 31)) as u32 as u64, "asr w"); - assert_eq!(cpu.x[5], ((a as i64) >> (b & 63)) as u64, "asr x"); - assert_eq!( - cpu.x[6], - ((a as i64 as i128 * b as i64 as i128) >> 64) as u64, - "smulh" - ); - assert_eq!(cpu.x[7], ((a as u128 * b as u128) >> 64) as u64, "umulh"); - assert_eq!(cpu.x[8], w as i32 as i64 as u64, "sxtw"); - assert_eq!(cpu.x[9], ((a >> 4) as u8) as i8 as i64 as u64, "sbfx"); - assert_eq!(cpu.x[10], (sw >> 3) as u32 as u64, "asr w imm"); - // Registers 11-17 come from the 32-bit compare, 18-20 from the 64-bit one. - let w_flags = [ - sw < swb, - sw >= swb, - w < wb, - w > wb, - sw > swb, - sw <= swb, - w <= wb, - ]; - for (i, want) in w_flags.into_iter().enumerate() { - assert_eq!(cpu.x[11 + i], want as u64, "w cond {i} {a:#x} {b:#x}"); - } - let (sa, sb) = (a as i64, b as i64); - assert_eq!(cpu.x[18], (sa < sb) as u64, "x lt"); - assert_eq!(cpu.x[19], (a < b) as u64, "x lo"); - assert_eq!(cpu.x[20], (sa > sb) as u64, "x gt"); - } - } - #[test] - fn signed_and_unsigned_vector_lanes_select_distinct_semantics() { - // smax.16b, sabd.8h, shadd.4s, sshl.4s, sqadd.16b, cmgt.2d, cmhi.2d. - let words = [ - 0x4e216403, 0x4e617404, 0x4ea10405, 0x4ea14406, 0x4e210c07, 0x4ee13409, 0x6ee1340a, - ]; - let block = compile(0, &bytes(&words)).unwrap(); - let a: [u8; 16] = [ - 0x80, 0x7f, 0x01, 0xff, 0x10, 0x90, 0x00, 0x7e, 0x85, 0x05, 0xfe, 0x40, 0x7f, 0x80, - 0xc0, 0x20, - ]; - let b: [u8; 16] = [ - 0x01, 0x80, 0xff, 0x01, 0x90, 0x10, 0x00, 0x7f, 0x05, 0x85, 0x02, 0xc0, 0x7f, 0x7f, - 0x40, 0xe0, - ]; - let pack = |x: [u8; 16]| { - let v = u128::from_le_bytes(x); - [v as u64, (v >> 64) as u64] - }; - let mut cpu = Cpu::default(); - cpu.v[0] = pack(a); - cpu.v[1] = pack(b); - block.run(&mut cpu); - let out = - |r: usize| (u128::from(cpu.v[r][1]) << 64 | u128::from(cpu.v[r][0])).to_le_bytes(); - let lanes16 = |x: [u8; 16]| -> Vec { - x.chunks(2) - .map(|c| i16::from_le_bytes([c[0], c[1]])) - .collect() - }; - let lanes32 = |x: [u8; 16]| -> Vec { - x.chunks(4) - .map(|c| i32::from_le_bytes([c[0], c[1], c[2], c[3]])) - .collect() - }; - // smax.16b - let want: Vec = (0..16) - .map(|i| (a[i] as i8).max(b[i] as i8) as u8) - .collect(); - assert_eq!(out(3).to_vec(), want, "smax"); - // sabd.8h - let want: Vec = lanes16(a) - .iter() - .zip(lanes16(b)) - .flat_map(|(x, y)| ((*x as i32 - y as i32).unsigned_abs() as u16).to_le_bytes()) - .collect(); - assert_eq!(out(4).to_vec(), want, "sabd"); - // shadd.4s - let want: Vec = lanes32(a) - .iter() - .zip(lanes32(b)) - .flat_map(|(x, y)| (((*x as i64 + y as i64) >> 1) as i32).to_le_bytes()) - .collect(); - assert_eq!(out(5).to_vec(), want, "shadd"); - // sshl.4s: the shift amount is the signed low byte of each lane of b. - let want: Vec = lanes32(a) - .iter() - .zip(lanes32(b)) - .flat_map(|(x, y)| { - let n = y as i8 as i32; - let r = if n >= 32 { - 0 - } else if n >= 0 { - x << n - } else if n > -32 { - x >> -n - } else if *x < 0 { - -1 - } else { - 0 - }; - r.to_le_bytes() - }) - .collect(); - assert_eq!(out(6).to_vec(), want, "sshl"); - // sqadd.16b - let want: Vec = (0..16) - .map(|i| (a[i] as i8).saturating_add(b[i] as i8) as u8) - .collect(); - assert_eq!(out(7).to_vec(), want, "sqadd"); - // cmgt.2d / cmhi.2d - for (r, signed) in [(9, true), (10, false)] { - for lane in 0..2 { - let (x, y) = (cpu.v[0][lane], cpu.v[1][lane]); - let gt = if signed { - (x as i64) > (y as i64) - } else { - x > y - }; - assert_eq!( - cpu.v[r][lane], - if gt { u64::MAX } else { 0 }, - "cmp {r} {lane}" - ); - } - } - } - #[test] fn pair_post_index_records_deferred_access_and_writeback() { // LDP X0,X1,[X2],#16. let block = compile(0x8000, &bytes(&[0xa8c10440])).unwrap(); @@ -3195,282 +363,4 @@ mod tests { run_inline(&mut cpu2, &[0xf9400001, SVC]); assert_eq!(cpu2.trap, Trap::Load); } - #[test] - fn simd_compare_zero_masks_each_lane() { - // cmeq v1.16b, v0.16b, #0. - let block = compile(0x1000, &bytes(&[0x4e209801])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[0][0] = 0x00ff_0000_0000_00ff; - cpu.v[0][1] = 0xffff_ffff_ffff_ffff; - block.run(&mut cpu); - assert_eq!(cpu.v[1][0], 0xff00_ffff_ffff_ff00); - assert_eq!(cpu.v[1][1], 0); - // Half-width form operates on v0.8b and clears the top half. - let block = compile(0x1000, &bytes(&[0x0e209801])).unwrap(); - cpu.v[0][0] = 0x00ff_0000_0000_00ff; - cpu.v[0][1] = 0x1234_5678_9abc_def0; - cpu.v[1] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; - block.run(&mut cpu); - assert_eq!(cpu.v[1][0], 0xff00_ffff_ffff_ff00); - assert_eq!(cpu.v[1][1], 0); - } - #[test] - fn simd_fmov_moves_bits_between_gpr_and_fpr() { - // fmov x2, d2 then fmov d3, x2 round-trips the low doubleword. - let block = compile(0x1000, &bytes(&[0x9e660042, 0x9e670043])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[2] = [0x1122_3344_5566_7788, 0xdead_beef_dead_beef]; - block.run(&mut cpu); - assert_eq!(cpu.x[2], 0x1122_3344_5566_7788); - assert_eq!(cpu.v[3], [0x1122_3344_5566_7788, 0]); - // 32-bit form truncates to the low word. - let block = compile(0x1000, &bytes(&[0x1e260022])).unwrap(); - cpu.v[1] = [0xaaaa_bbbb_cccc_dddd, 0]; - block.run(&mut cpu); - assert_eq!(cpu.x[2] as u32, 0xcccc_dddd); - } - #[test] - fn simd_pairwise_reductions_match_lane_layout() { - let setup = || { - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; - cpu.v[2] = [0x1112_1314_1516_1718, 0x191a_1b1c_1d1e_1f20]; - cpu - }; - for (word, want) in [ - ( - 0x6e22a420u32, - [0x0a0c_0e10_0204_0608u64, 0x1a1c_1e20_1214_1618], - ), - (0x6e22ac20, [0x090b_0d0f_0103_0507, 0x191b_1d1f_1113_1517]), - (0x4e22bc20, [0x1317_1b1f_0307_0b0f, 0x3337_3b3f_2327_2b2f]), - (0x4ea2bc20, [0x1618_1a1c_0608_0a0c, 0x3638_3a3c_2628_2a2c]), - (0x4ee2bc20, [0x0a0c_0e10_1214_1618, 0x2a2c_2e30_3234_3638]), - ] { - let mut cpu = setup(); - let block = compile(0x1000, &bytes(&[word])).unwrap(); - block.run(&mut cpu); - assert_eq!(cpu.v[0], want, "{word:08x}"); - } - } - #[test] - fn simd_horizontal_add_sums_lanes() { - for (word, lanes, want) in [ - (0x4e31b820u32, 16usize, 0x88u64), - (0x4e71b820, 8, 0x4048), - (0x4eb1b820, 4, 0x1c20_2428), - ] { - let block = compile(0x1000, &bytes(&[word])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; - block.run(&mut cpu); - let _ = lanes; - assert_eq!(cpu.v[0], [want, 0], "{word:08x}"); - } - // The 64-bit form reads only the low half of the source. - let block = compile(0x1000, &bytes(&[0x0e31b820])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; - block.run(&mut cpu); - assert_eq!(cpu.v[0], [0x24, 0]); - } - #[test] - fn simd_dup_from_element_replicates_a_lane() { - for (word, want) in [ - // dup v0.4h, v1.h[0] -> element 0x0708 in every halfword. - (0x0e020420u32, [0x0708_0708_0708_0708u64, 0]), - // dup v0.8h, v1.h[7] -> element 0x090a, both halves. - (0x4e1e0420, [0x090a_090a_090a_090a, 0x090a_090a_090a_090a]), - // dup v0.4s, v1.s[1] -> element 0x0102_0304. - (0x4e0c0420, [0x0102_0304_0102_0304, 0x0102_0304_0102_0304]), - // dup v0.2d, v1.d[1] -> element 0x090a_0b0c_0d0e_0f10. - (0x4e180420, [0x090a_0b0c_0d0e_0f10, 0x090a_0b0c_0d0e_0f10]), - // dup v0.16b, v1.b[3] -> element 0x05. - (0x4e070420, [0x0505_0505_0505_0505, 0x0505_0505_0505_0505]), - ] { - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; - let block = compile(0x1000, &bytes(&[word])).unwrap(); - block.run(&mut cpu); - assert_eq!(cpu.v[0], want, "{word:08x}"); - } - } - #[test] - fn simd_table_lookup_selects_bytes() { - // tbl v0.16b, {v1.16b, v2.16b}, v3.16b with indices 0,2,4,... - let block = compile(0x1000, &bytes(&[0x4e032020])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; - cpu.v[2] = [0x1716_1514_1312_1110, 0x1f1e_1d1c_1b1a_1918]; - cpu.v[3] = [0x0604_0200_0e0c_0a08, 0x1614_1210_1e1c_1a18]; - block.run(&mut cpu); - assert_eq!(cpu.v[0], [0x0604_0200_0e0c_0a08, 0x1614_1210_1e1c_1a18]); - // Out-of-range indices yield zero for tbl. - let block = compile(0x1000, &bytes(&[0x4e020020])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; - cpu.v[2] = [0xff01_ff00_ff00_ff00, 0xffff_ffff_ffff_ffff]; - block.run(&mut cpu); - assert_eq!(cpu.v[0], [0x0001_0000_0000_0000, 0]); - // tbx keeps the destination byte where the index is out of range. - let block = compile(0x1000, &bytes(&[0x4e021020])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; - cpu.v[2] = [0xff01_ff00_ff00_ff00, 0xffff_ffff_ffff_ffff]; - cpu.v[0] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; - block.run(&mut cpu); - assert_eq!(cpu.v[0], [0xde01_be00_de00_be00, 0xdead_beef_dead_beef]); - } - #[test] - fn simd_alu_lane_semantics() { - // Ground truth computed with the host's own byte arithmetic. - struct Case { - word: u32, - a: [u64; 2], - b: [u64; 2], - want: [u64; 2], - } - let cases = [ - // add v0.16b, v1.16b, v2.16b wraps per byte - Case { - word: 0x4e228420, - a: [0x0102_03ff_0405_0607, 0x0f10_1120_2122_2324], - b: [0x0101_0101_0101_0101, 0x0102_0304_0506_0708], - want: [0x0203_0400_0506_0708, 0x1012_1424_2628_2a2c], - }, - // sub v0.16b, v1.16b, v2.16b - Case { - word: 0x6e228420, - a: [0x0001_0203_0405_0607, 0], - b: [0x0101_0101_0101_0101, 0], - want: [0xff00_0102_0304_0506, 0], - }, - // cmhs v0.16b, v1.16b, v2.16b (unsigned >=) - Case { - word: 0x6e223c20, - a: [0x00ff_0080_7f7f_8000, 0], - b: [0x0001_0180_7f80_8000, 0], - want: [0xffff_00ff_ff00_ffff, 0xffff_ffff_ffff_ffff], - }, - // umin v0.8h, v1.8h, v2.8h - Case { - word: 0x6e626c20, - a: [0x0003_0002_0001_0000, 0], - b: [0x0002_0003_0000_0001, 0], - want: [0x0002_0002_0000_0000, 0], - }, - // uqadd v0.8h, v1.8h, v2.8h saturates - Case { - word: 0x6e620c20, - a: [0xffff_8000_0001_0000, 0], - b: [0x0001_9000_0002_0001, 0], - want: [0xffff_ffff_0003_0001, 0], - }, - // shadd v0.16b, v1.16b, v2.16b rounds down (arithmetic) - Case { - word: 0x4e220420, - a: [0x0303_fefe_0505_00ff, 0], - b: [0x0303_fefe_0505_00ff, 0], - want: [0x0303_fefe_0505_00ff, 0], - }, - // sshl v0.8h by signed amounts in the low byte of each lane - Case { - word: 0x4e624420, - a: [0x0004_0003_0002_0001, 0], - b: [0x0000_0002_00ff_0001, 0], - want: [0x0004_000c_0001_0002, 0], - }, - // mla/mls fold in the destination *lane*, not the whole half - Case { - word: 0x4e229420, - a: [0x0202_0202_0202_0202, 0], - b: [0x0505_0505_0505_0505, 0], - want: [0x0a0a_0a0a_0a0a_0a0a, 0], - }, - Case { - word: 0x6e229420, - a: [0x0202_0202_0202_0202, 0], - b: [0x0505_0505_0505_0505, 0], - want: [0xf6f6_f6f6_f6f6_f6f6, 0], - }, - // bsl: destination starts zero, so the result is v2 unchanged - Case { - word: 0x6e621c20, - a: [0xaaaa_aaaa_aaaa_aaaa, 0], - b: [0x5555_5555_5555_5555, 0], - want: [0x5555_5555_5555_5555, 0], - }, - // bit with a zero destination inserts the masked v1 bits - Case { - word: 0x6ea21c20, - a: [0x0f0f_0f0f_0f0f_0f0f, 0], - b: [0xaaaa_aaaa_aaaa_aaaa, 0], - want: [0x0a0a_0a0a_0a0a_0a0a, 0], - }, - // bif with a zero destination inserts the inverse-masked bits - Case { - word: 0x6ee21c20, - a: [0x0f0f_0f0f_0f0f_0f0f, 0], - b: [0xaaaa_aaaa_aaaa_aaaa, 0], - want: [0x0505_0505_0505_0505, 0], - }, - // shadd of two -1 bytes stays -1 (arithmetic halving) - Case { - word: 0x4e220420, - a: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], - b: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], - want: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], - }, - // shsub of 0 - 1 halves to -1 - Case { - word: 0x4e222420, - a: [0, 0], - b: [0x0101_0101_0101_0101, 0], - want: [0xffff_ffff_ffff_ffff, 0], - }, - // srhadd of 1 and 2 rounds to 2 - Case { - word: 0x4e221420, - a: [0x0101_0101_0101_0101, 0], - b: [0x0202_0202_0202_0202, 0], - want: [0x0202_0202_0202_0202, 0], - }, - // sshl v0.8h: lane 0 shifts left by 1, lane 1 right by 1 - Case { - word: 0x4e624420, - a: [0x0008_0004_0002_0001, 0], - b: [0x0000_00ff_0001_0001, 0], - want: [0x0008_0002_0004_0002, 0], - }, - // uqshl v0.8h saturates an overflow to the lane maximum - Case { - word: 0x6e624c20, - a: [0x0000_0000_0000_8000, 0], - b: [0x0000_0000_0000_0001, 0], - want: [0x0000_0000_0000_ffff, 0], - }, - // sqshl v0.8h saturates a positive overflow to the lane maximum - Case { - word: 0x4e624c20, - a: [0x0000_0000_0000_4000, 0], - b: [0x0000_0000_0000_0001, 0], - want: [0x0000_0000_0000_7fff, 0], - }, - // sqrshl v0.8h rounds before shifting right - Case { - word: 0x4e625c20, - a: [0x0000_0000_0000_0003, 0], - b: [0x0000_0000_0000_00ff, 0], - want: [0x0000_0000_0000_0002, 0], - }, - ]; - for case in cases { - let block = compile(0x1000, &bytes(&[case.word])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = case.a; - cpu.v[2] = case.b; - block.run(&mut cpu); - assert_eq!(cpu.v[0], case.want, "{:08x}", case.word); - } - } } diff --git a/src/emulator/src/aarch64/environment.rs b/src/emulator/src/aarch64/environment.rs new file mode 100644 index 0000000..c98c735 --- /dev/null +++ b/src/emulator/src/aarch64/environment.rs @@ -0,0 +1,1037 @@ +//! The emulator's [`Environment`]: guest state is the [`Cpu`] struct, memory goes +//! through the inline data TLB and the trap-and-resume slow path, and system +//! instructions read and write [`CpuSystem`]. +use super::cpu::{Cpu, DTLB_ENTRIES, Exclusive as CpuExclusive, System as CpuSystem, Trap}; +use core::mem::offset_of; +use volt_ir::function::{BinOp as B, CmpOp as C, Value}; +use volt_lift_aarch64::{Builder, Environment, LiftError, MemoryAccess, Writeback}; +use volt_target::aarch64::decode::*; + +/// Lowers guest instructions against [`Cpu`]: the lifter's machine policy. +pub(super) struct CpuEnvironment { + /// Plain loads and stores get an inline data-TLB fast path and do not end + /// the block; the block scan must use `Instruction::terminates_with(true)`. + inline_memory: bool, + /// Guest instructions started but not yet added to `cntvct_el0`. The + /// counter is bumped once per exit path (`exit`) and before anything that + /// reads it, instead of with a load/add/store per instruction. + pending: u64, +} +impl CpuEnvironment { + pub(super) fn new(inline_memory: bool) -> Self { + Self { + inline_memory, + pending: 0, + } + } + /// A word the host runs (see `host::run`) instead of the lifter: it + /// counts as a retired instruction, hands the raw word to the machine, and + /// ends the block. + pub(super) fn host(&mut self, b: &Builder, pc: u64, word: u32) { + self.pending += 1; + b.store(offset_of!(Cpu, value), b.k(b.i64, word as u64)); + self.trap(b, pc.wrapping_add(4), Trap::Host); + } + fn add_counter(&self, b: &Builder, n: u64) { + let at = offset_of!(Cpu, system) + offset_of!(CpuSystem, cntvct_el0); + b.store(at, b.imm(b.i64, B::Add, b.load(b.i64, at), n)); + } + /// Make `cntvct_el0` in memory current on the path being emitted. + fn flush_counter(&mut self, b: &Builder) { + let n = core::mem::replace(&mut self.pending, 0); + if n != 0 { + self.add_counter(b, n); + } + } + /// Leave the function with the counter current. `pending` is not cleared: + /// sibling paths (slow blocks) share the main path's count. + fn leave(&self, b: &Builder, next_pc: Value) { + if self.pending != 0 { + self.add_counter(b, self.pending); + } + b.set_ret(next_pc) + } + /// Hand the base-register update of an access to the machine, which applies + /// it once the access has completed. + fn defer_writeback(&self, b: &Builder, w: Writeback) { + b.store(offset_of!(Cpu, writeback_value), w.value); + b.byte(offset_of!(Cpu, writeback_dest), w.register as u64); + b.byte(offset_of!(Cpu, writeback), 1); + } + + /// Probe the inline data TLB for `address`. Returns `(hit, host_pointer)`; + /// the pointer is only meaningful when `hit`. A hit requires the page to be + /// cached for this access kind and the access to be naturally aligned (so it + /// cannot cross a page), because the tag compare covers the low `size - 1` + /// address bits too. + fn dtlb_probe(&self, b: &Builder, address: Value, size: Size, write: bool) -> (Value, Value) { + let n = size as u64; + // Entry offset = ((address >> 12) & (ENTRIES - 1)) * 32 = (address >> 7) & mask. + let index = b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, address, 7), + (DTLB_ENTRIES as u64 - 1) << 5, + ); + let entry = b.bin(b.ptr, B::Add, b.state(), index); + // Constant displacements fold into the load addressing. + let field = |at: usize| { + b.load_ptr( + b.i64, + b.imm(b.ptr, B::Add, entry, (offset_of!(Cpu, dtlb) + at) as u64), + ) + }; + let tag = field(if write { 8 } else { 0 }); + let addend = field(16); + let host = b.bin(b.ptr, B::Add, address, addend); + let want = b.imm(b.i64, B::BitAnd, address, !0xfffu64 | (n - 1)); + // Last, so the backend can fuse it into the branch that consumes it. + let hit = b.cmp(C::Eq, tag, want); + (hit, host) + } + fn fast_load(&self, b: &Builder, host: Value, size: Size, rt: u8, signed: SignExtend) { + let bits = size as u16 * 8; + let raw = b.load_ptr(b.int(bits), host); + if rt == 31 { + return; + } + let mut v = b.cv(b.i64, raw); + if signed != SignExtend::None && bits < 64 { + let n = 64 - bits as u64; + let extended = b.shift(Width::X64, b.imm(b.i64, B::Shl, v, n), Shift::Asr, n); + v = if signed == SignExtend::To32 { + b.imm(b.i64, B::BitAnd, extended, 0xffff_ffff) + } else { + extended + }; + } + b.put(rt, v); + } + fn fast_store(&self, b: &Builder, host: Value, size: Size, rt: u8) { + let t = b.int(size as u16 * 8); + b.store_ptr(host, b.cv(t, b.reg(b.i64, rt, true))); + } + /// Emit a load or store with an inline fast path. On a data-TLB hit the + /// access happens in place and emission continues in a fresh block; on a + /// miss control goes to a slow block that hands the access to the machine + /// (`slow` fills in the request) and returns. + fn memory_access( + &self, + b: &Builder, + address: Value, + size: Size, + op: MemoryOp, + rt: u8, + signed: SignExtend, + post: Option, + slow: impl FnOnce(&Self), + ) { + let write = op == MemoryOp::Store; + let (hit, host) = self.dtlb_probe(b, address, size, write); + let (fast_block, slow_block) = (b.new_block(), b.new_block()); + b.branch_if(hit, fast_block, slow_block); + b.set_block(slow_block); + slow(self); + b.set_block(fast_block); + if write { + self.fast_store(b, host, size, rt); + } else { + self.fast_load(b, host, size, rt, signed); + } + if let Some(w) = post { + b.put_sp(w.register, w.value); + } + } + /// Inline fast path for a GPR load/store pair: one data-TLB probe covers + /// both accesses when the first is naturally aligned and the second starts + /// in the same page. Anything else takes `slow`, which hands the whole pair + /// to the machine. + fn pair_access( + &self, + b: &Builder, + address: Value, + a: Pair, + post: Option, + slow: impl FnOnce(&Self), + ) { + let write = a.op == MemoryOp::Store; + let n = a.size as u64; + let (hit, host) = self.dtlb_probe(b, address, a.size, write); + let (check_block, fast_block, slow_block) = (b.new_block(), b.new_block(), b.new_block()); + b.branch_if(hit, check_block, slow_block); + // `address` is aligned to `n`, so the second access leaves the page + // exactly when it starts on a page boundary. + b.set_block(check_block); + let second_at = b.imm(b.i64, B::Add, address, n); + let offset = b.imm(b.i64, B::BitAnd, second_at, 0xfff); + let same_page = b.cmp(C::Ne, offset, b.k(b.i64, 0)); + b.branch_if(same_page, fast_block, slow_block); + b.set_block(slow_block); + slow(self); + b.set_block(fast_block); + let second = b.imm(b.ptr, B::Add, host, n); + if write { + self.fast_store(b, host, a.size, a.rt); + self.fast_store(b, second, a.size, a.rt2); + } else { + self.fast_load(b, host, a.size, a.rt, a.signed); + self.fast_load(b, second, a.size, a.rt2, a.signed); + } + if let Some(w) = post { + b.put_sp(w.register, w.value); + } + } + fn trap(&self, b: &Builder, pc: u64, t: Trap) { + b.byte(offset_of!(Cpu, trap), t as u64); + self.leave(b, b.k(b.i64, pc)) + } + + /// Slow path for a whole SIMD vector or vector-pair access: describe the + /// full instruction in the CPU request fields (`vector_dest`, optionally + /// `second_*`) and trap out; the machine completes every byte and resumes + /// after the instruction. + fn simd_trap_access( + &self, + b: &Builder, + address: Value, + bytes: u64, + rt: u8, + rt2: Option, + post: Option, + pc: u64, + trap: Trap, + ) { + b.store(offset_of!(Cpu, address), address); + b.byte(offset_of!(Cpu, width), bytes); + b.byte(offset_of!(Cpu, dest), rt as u64); + b.byte(offset_of!(Cpu, vector_dest), 1); + if let Some(rt2) = rt2 { + let second = b.imm(b.i64, B::Add, address, if bytes == 16 { 16 } else { 8 }); + b.store(offset_of!(Cpu, second_address), second); + b.byte(offset_of!(Cpu, second_width), bytes); + b.byte(offset_of!(Cpu, second_dest), rt2 as u64); + b.byte(offset_of!(Cpu, second_vector), 1); + b.byte(offset_of!(Cpu, second_pending), 1); + } + if let Some(w) = post { + self.defer_writeback(b, w); + } + self.trap(b, pc.wrapping_add(4), trap); + } + /// Describe a GPR pair access to the machine: the first access in the usual + /// request fields, the second as `second_*`, completed after the first. + fn setup_pair(&self, b: &Builder, address: Value, a: Pair) { + self.setup_memory(b, address, a.size, a.rt, a.signed, a.op); + b.store( + offset_of!(Cpu, second_address), + b.imm(b.i64, B::Add, address, a.size as u64), + ); + b.byte(offset_of!(Cpu, second_width), a.size as u64); + b.byte(offset_of!(Cpu, second_dest), a.rt2 as u64); + if a.signed != SignExtend::None { + b.byte(offset_of!(Cpu, second_signed), a.signed as u64) + } + if a.op == MemoryOp::Store { + b.store(offset_of!(Cpu, second_value), b.reg(b.i64, a.rt2, true)) + } + b.byte(offset_of!(Cpu, second_pending), 1); + } + fn setup_memory( + &self, + b: &Builder, + address: Value, + size: Size, + rt: u8, + signed: SignExtend, + op: MemoryOp, + ) { + b.store(offset_of!(Cpu, address), address); + b.byte(offset_of!(Cpu, width), size as u64); + b.byte(offset_of!(Cpu, dest), rt as u64); + if signed != SignExtend::None { + b.byte(offset_of!(Cpu, load_signed), signed as u64) + } + if op == MemoryOp::Store { + b.store(offset_of!(Cpu, value), b.reg(b.i64, rt, true)) + } + } + + /// Apply a base-register update the way a non-inline access does: a + /// pre-index update immediately, a post-index one via the machine. + fn writeback_eager(&self, b: &Builder, post: Option) { + match post { + Some(w) if w.pre_index => b.put_sp(w.register, w.value), + Some(w) => self.defer_writeback(b, w), + None => {} + } + } + fn scalar(&self, b: &Builder, pc: u64, a: Memory, address: Value, post: Option) { + let trap = if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }; + if self.inline_memory { + self.memory_access(b, address, a.size, a.op, a.rt, a.signed, post, |env| { + env.setup_memory(b, address, a.size, a.rt, a.signed, a.op); + if let Some(w) = post { + env.defer_writeback(b, w); + } + env.trap(b, pc.wrapping_add(4), trap); + }); + } else { + self.writeback_eager(b, post); + self.setup_memory(b, address, a.size, a.rt, a.signed, a.op); + self.trap(b, pc.wrapping_add(4), trap); + } + } + fn literal(&self, b: &Builder, pc: u64, a: Literal, address: Value) { + if self.inline_memory { + self.memory_access( + b, + address, + a.size, + MemoryOp::Load, + a.rt, + SignExtend::None, + None, + |env| { + env.setup_memory(b, address, a.size, a.rt, SignExtend::None, MemoryOp::Load); + env.trap(b, pc.wrapping_add(4), Trap::Load); + }, + ); + } else { + self.setup_memory(b, address, a.size, a.rt, SignExtend::None, MemoryOp::Load); + self.trap(b, pc.wrapping_add(4), Trap::Load); + } + } + fn pair(&self, b: &Builder, pc: u64, a: Pair, address: Value, post: Option) { + let trap = if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }; + if self.inline_memory { + self.pair_access(b, address, a, post, |env| { + env.setup_pair(b, address, a); + if let Some(w) = post { + env.defer_writeback(b, w); + } + env.trap(b, pc.wrapping_add(4), trap); + }); + } else { + self.writeback_eager(b, post); + self.setup_pair(b, address, a); + self.trap(b, pc.wrapping_add(4), trap); + } + } + fn simd_pair( + &self, + b: &Builder, + pc: u64, + a: SimdPair, + address: Value, + post: Option, + ) { + let trap = if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }; + if !self.inline_memory { + self.simd_trap_access( + b, + address, + a.bytes as u64, + a.rt, + Some(a.rt2), + post, + pc, + trap, + ); + } else { + let at_rt = b.v_at(a.rt); + let at_rt2 = b.v_at(a.rt2); + let slow_block = b.new_block(); + let store = a.op == MemoryOp::Store; + let (bytes, chunk_count) = if a.bytes == 16 { (16, 4) } else { (8, 2) }; + let starts: [Value; 4] = [ + address, + b.imm(b.i64, B::Add, address, 8), + b.imm(b.i64, B::Add, address, 16), + b.imm(b.i64, B::Add, address, 24), + ]; + // stores: (reg_offset_base, half) per chunk + let src: [(usize, u64); 4] = if bytes == 16 { + [(at_rt, 0), (at_rt + 8, 0), (at_rt2, 0), (at_rt2 + 8, 0)] + } else { + [(at_rt, 0), (at_rt2, 0), (at_rt, 0), (at_rt, 0)] + }; + let mut hosts: [Option; 4] = [None; 4]; + for i in 0..chunk_count { + let next = b.new_block(); + let (hit, host) = self.dtlb_probe(b, starts[i], Size::Double, store); + b.branch_if(hit, next, slow_block); + b.set_block(next); + hosts[i] = Some(host); + } + if store { + for i in 0..chunk_count { + let host = hosts[i].unwrap(); + b.store_ptr(host, b.load(b.i64, src[i].0)); + } + } else { + for i in 0..chunk_count { + let host = hosts[i].unwrap(); + b.store(src[i].0, b.load_ptr(b.i64, host)); + } + // zero the untransferred half of each 128-bit register + if bytes == 8 { + b.store(at_rt + 8, b.k(b.i64, 0)); + b.store(at_rt2 + 8, b.k(b.i64, 0)); + } + } + if let Some(w) = post { + b.put_sp(w.register, w.value); + } + // slow path, built last so every fast edge into it exists. + let resume = b.block(); + b.set_block(slow_block); + self.simd_trap_access(b, address, bytes as u64, a.rt, Some(a.rt2), post, pc, trap); + b.set_block(resume); + } + } + fn simd_memory( + &self, + b: &Builder, + pc: u64, + a: SimdMemory, + address: Value, + post: Option, + ) { + let trap = if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }; + if !self.inline_memory { + self.simd_trap_access(b, address, a.bytes as u64, a.rt, None, post, pc, trap); + } else { + let at_rt = b.v_at(a.rt); + let slow_block = b.new_block(); + let store = a.op == MemoryOp::Store; + // One chunk for widths up to 8 bytes, two 8-byte chunks for 16. + let wide = a.bytes == 16; + let size = match a.bytes { + 1 => Size::Byte, + 2 => Size::Half, + 4 => Size::Word, + _ => Size::Double, + }; + let chunk_count = if wide { 2 } else { 1 }; + let starts: [Value; 2] = [address, b.imm(b.i64, B::Add, address, 8)]; + let chunk_ty = b.int((a.bytes.min(8) as u16) * 8); + let src: [usize; 2] = [at_rt, at_rt + 8]; + let mut hosts: [Option; 2] = [None; 2]; + for i in 0..chunk_count { + let next = b.new_block(); + let (hit, host) = self.dtlb_probe(b, starts[i], size, store); + b.branch_if(hit, next, slow_block); + b.set_block(next); + hosts[i] = Some(host); + } + if store { + for i in 0..chunk_count { + let host = hosts[i].unwrap(); + b.store_ptr(host, b.cv(chunk_ty, b.load(b.i64, src[i]))); + } + } else { + for i in 0..chunk_count { + let host = hosts[i].unwrap(); + b.store(src[i], b.cv(b.i64, b.load_ptr(chunk_ty, host))); + } + // Scalar loads zero the whole upper register, not just the + // untransferred half. + if a.bytes <= 8 { + b.store(at_rt + 8, b.k(b.i64, 0)); + if a.bytes < 8 { + let kept = (1u64 << (a.bytes * 8)) - 1; + let lo = b.imm(b.i64, B::BitAnd, b.load(b.i64, at_rt), kept); + b.store(at_rt, lo); + } + } + } + if let Some(w) = post { + b.put_sp(w.register, w.value); + } + let resume = b.block(); + b.set_block(slow_block); + self.simd_trap_access(b, address, a.bytes as u64, a.rt, None, post, pc, trap); + b.set_block(resume); + } + } + fn exclusive(&self, b: &Builder, pc: u64, a: Exclusive) { + self.setup_memory( + b, + b.reg(b.i64, a.rn, false), + a.size, + a.rt, + SignExtend::None, + a.op, + ); + if a.op == MemoryOp::Store { + b.byte(offset_of!(Cpu, status_dest), a.rs as u64) + } + b.byte( + offset_of!(Cpu, exclusive), + if a.op == MemoryOp::Store { + CpuExclusive::Store as u64 + } else { + CpuExclusive::Load as u64 + }, + ); + self.trap( + b, + pc.wrapping_add(4), + if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }, + ); + } + fn atomic(&self, b: &Builder, pc: u64, a: Atomic) { + // Read-modify-write needs a host atomic, so it always takes the machine + // slow path. `dests` receive the words read (31 discards) and `operands` + // are what the machine combines with memory (see `AtomicKind::apply`). + let (dests, operands): ([u8; 2], &[u8]) = match a.kind { + AtomicKind::Casp => ([a.rs, a.rs + 1], &[a.rs, a.rs + 1, a.rt, a.rt + 1]), + AtomicKind::Cas => ([a.rs, 31], &[a.rs, a.rt]), + AtomicKind::LoadPair => ([a.rt, a.rt2], &[]), + AtomicKind::StorePair => ([31, 31], &[a.rt, a.rt2]), + _ => ([a.rt, 31], &[a.rs]), + }; + b.store(offset_of!(Cpu, address), b.reg(b.i64, a.rn, false)); + b.byte(offset_of!(Cpu, width), a.size as u64); + b.byte(offset_of!(Cpu, dest), dests[0] as u64); + b.byte(offset_of!(Cpu, atomic_kind), a.kind as u64); + for (slot, &dest) in dests.iter().enumerate() { + b.byte(offset_of!(Cpu, atomic_dests) + slot, dest as u64); + } + for (slot, ®) in operands.iter().enumerate() { + // Register numbers past 30 read as XZR. + b.store( + offset_of!(Cpu, atomic_operands) + slot * 8, + b.reg(b.i64, reg, true), + ); + } + if a.kind == AtomicKind::StorePair { + b.byte(offset_of!(Cpu, status_dest), a.rs as u64); + } + self.trap(b, pc.wrapping_add(4), Trap::Atomic); + } + fn simd_struct(&self, b: &Builder, pc: u64, a: SimdStruct) { + use volt_target::aarch64::simd_struct::StructDesc; + // Element-wise accesses with lane merging need the machine slow path. + let d = a.desc; + let address = b.reg(b.i64, a.rn, false); + b.store(offset_of!(Cpu, address), address); + for (field, value) in [ + (offset_of!(StructDesc, store), u64::from(d.store)), + (offset_of!(StructDesc, shape), d.shape as u64), + (offset_of!(StructDesc, registers), u64::from(d.registers)), + ( + offset_of!(StructDesc, interleaved), + u64::from(d.interleaved), + ), + (offset_of!(StructDesc, esize), u64::from(d.esize)), + (offset_of!(StructDesc, lane), u64::from(d.lane)), + (offset_of!(StructDesc, q), u64::from(d.q)), + (offset_of!(StructDesc, rt), u64::from(d.rt)), + ] { + b.byte(offset_of!(Cpu, structure) + field, value); + } + let step = match a.post { + StructPost::None => None, + StructPost::Immediate => Some(b.k(b.i64, d.bytes())), + StructPost::Register(rm) => Some(b.reg(b.i64, rm, true)), + }; + if let Some(step) = step { + b.store( + offset_of!(Cpu, writeback_value), + b.bin(b.i64, B::Add, address, step), + ); + b.byte(offset_of!(Cpu, writeback_dest), a.rn as u64); + b.byte(offset_of!(Cpu, writeback), 1); + } + self.trap(b, pc.wrapping_add(4), Trap::SimdStruct); + } + + fn lower_system(&mut self, b: &Builder, a: System) { + use SystemRegister::*; + self.flush_counter(b); + let at = offset_of!(Cpu, system); + let level = match a.register { + SpEl0 => Some(0u64), + SpEl1 => Some(1), + SpEl2 => Some(2), + _ => None, + }; + if let Some(level) = level { + let current = b.cv(b.i64, b.load(b.i8, at + offset_of!(CpuSystem, el))); + let at_level = b.cmp(C::Eq, current, b.k(b.i64, level)); + let live = if level <= 1 { + let el1 = b.cmp(C::Eq, current, b.k(b.i64, 1)); + let selected = b.cmp( + if level == 0 { C::Eq } else { C::Ne }, + b.load(b.i8, at + offset_of!(CpuSystem, spsel)), + b.k(b.i8, 0), + ); + let el1_selected = b.bin(b.boolean, B::BitAnd, el1, selected); + if level == 0 { + b.bin(b.boolean, B::BitOr, at_level, el1_selected) + } else { + el1_selected + } + } else { + at_level + }; + let p = b.sel( + b.ptr, + live, + b.addr(b.reg_at(31)), + b.addr(at + offset_of!(CpuSystem, sp_el) + level as usize * 8), + ); + if a.read { + b.put(a.rt, b.load_ptr(b.i64, p)) + } else { + b.store_ptr(p, b.reg(b.i64, a.rt, true)) + } + return; + } + if let Some((el1, el2)) = vhe_banks(a.register) { + // HCR_EL2.E2H: EL1 register names accessed from EL2 reach the EL2 banks. + let current = b.cv(b.i64, b.load(b.i8, at + offset_of!(CpuSystem, el))); + let el2_mode = b.cmp(C::Eq, current, b.k(b.i64, 2)); + let e2h = b.cmp( + C::Ne, + b.imm( + b.i64, + B::BitAnd, + b.load(b.i64, at + offset_of!(CpuSystem, hcr_el2)), + super::cpu::HCR_E2H, + ), + b.k(b.i64, 0), + ); + let redirect = b.bin(b.boolean, B::BitAnd, el2_mode, e2h); + let p = b.sel(b.ptr, redirect, b.addr(at + el2), b.addr(at + el1)); + if a.read { + b.put(a.rt, b.load_ptr(b.i64, p)); + } else { + b.store_ptr(p, b.reg(b.i64, a.rt, true)); + } + return; + } + match a.register { + CurrentEl => b.put( + a.rt, + b.imm( + b.i64, + B::Shl, + b.cv(b.i64, b.load(b.i8, at + offset_of!(CpuSystem, el))), + 2, + ), + ), + Pan | PanSet => { + // PSTATE.PAN lives in bit 22 of the register form. + let pan = at + offset_of!(CpuSystem, pan); + if a.read { + b.put(a.rt, b.imm(b.i64, B::Shl, b.load(b.i64, pan), 22)); + } else if a.register == PanSet { + b.store(pan, b.k(b.i64, u64::from(a.immediate & 1))); + } else { + let bit = b.imm(b.i64, B::Shr, b.reg(b.i64, a.rt, true), 22); + b.store(pan, b.imm(b.i64, B::BitAnd, bit, 1)); + } + } + RazWi | AppleZero => { + if a.read { + b.put(a.rt, b.k(b.i64, 0)); + } + } + SpSel | SpSelSet => { + let selected = b.load(b.i8, at + offset_of!(CpuSystem, spsel)); + if a.read { + b.put(a.rt, b.cv(b.i64, selected)); + return; + } + let new = if a.register == SpSelSet { + b.k(b.i8, u64::from(a.immediate & 1)) + } else { + b.cv(b.i8, b.imm(b.i64, B::BitAnd, b.reg(b.i64, a.rt, true), 1)) + }; + // The live stack pointer belongs to bank `SP_EL1` when SPSel is 1 + // and `SP_EL0` when it is 0. Park it in its bank, then load the + // other one. Selecting the bank already in use is a store and a + // reload of the same slot. + let banks = at + offset_of!(CpuSystem, sp_el); + let bank = |select: Value| { + b.sel( + b.ptr, + b.cmp(C::Ne, select, b.k(b.i8, 0)), + b.addr(banks + 8), + b.addr(banks), + ) + }; + let live = b.load(b.i64, b.reg_at(31)); + b.store_ptr(bank(selected), live); + let incoming = b.load_ptr(b.i64, bank(new)); + b.store(b.reg_at(31), incoming); + b.store(at + offset_of!(CpuSystem, spsel), new); + } + DaifSet | DaifClear => { + let p = at + offset_of!(CpuSystem, daif); + let before = b.load(b.i64, p); + let updated = b.imm( + b.i64, + if a.register == DaifSet { + B::BitOr + } else { + B::BitAnd + }, + before, + if a.register == DaifSet { + a.immediate as u64 + } else { + (a.immediate as u64) ^ 0xf + }, + ); + b.store(p, updated); + } + Nzcv => { + if a.read { + b.put(a.rt, b.cv(b.i64, b.load(b.i32, offset_of!(Cpu, flags)))) + } else { + b.store( + offset_of!(Cpu, flags), + b.imm( + b.i32, + B::BitAnd, + b.cv(b.i32, b.reg(b.i64, a.rt, true)), + 0xf0000000, + ), + ) + } + } + OslsrEl1 => { + let either = b.bin( + b.i64, + B::BitOr, + b.load(b.i64, at + offset_of!(CpuSystem, oslar_el1)), + b.load(b.i64, at + offset_of!(CpuSystem, osdlr_el1)), + ); + let locked = b.cmp(C::Ne, either, b.k(b.i64, 0)); + b.put(a.rt, b.sel(b.i64, locked, b.k(b.i64, 2), b.k(b.i64, 0))); + } + CcsidrEl1 => { + // CSSELR bit 0 is InD, bits 3:1 the level - 1; L2 has no separate I-side. + use super::identification::{CCSIDR_L1D, CCSIDR_L1I, CCSIDR_L2}; + let select = b.imm( + b.i64, + B::BitAnd, + b.load(b.i64, at + offset_of!(CpuSystem, csselr_el1)), + 0xf, + ); + let is = |n: u64| b.cmp(C::Eq, select, b.k(b.i64, n)); + let l1_instruction = + b.sel(b.i64, is(1), b.k(b.i64, CCSIDR_L1I), b.k(b.i64, CCSIDR_L2)); + b.put( + a.rt, + b.sel(b.i64, is(0), b.k(b.i64, CCSIDR_L1D), l1_instruction), + ); + } + CntvTvalEl0 | CntpTvalEl0 => { + // The timer value is the signed 32-bit distance to the compare value: + // reads give the low 32 bits of `CVAL - count`, writes `CVAL = count + sext(value)`. + // The physical timer shares the virtual count, as CNTVOFF_EL2 is zero without EL2. + let counter = b.load(b.i64, at + offset_of!(CpuSystem, cntvct_el0)); + let compare = at + + if a.register == CntpTvalEl0 { + offset_of!(CpuSystem, cntp_cval_el0) + } else { + offset_of!(CpuSystem, cntv_cval_el0) + }; + if a.read { + let distance = b.bin(b.i64, B::Sub, b.load(b.i64, compare), counter); + b.put(a.rt, b.imm(b.i64, B::BitAnd, distance, 0xffff_ffff)); + } else { + let delta = b.extend(b.reg(b.i64, a.rt, true), Extend::Sxtw); + b.store(compare, b.bin(b.i64, B::Add, counter, delta)); + } + } + CntpctEl0 => { + // Writes are ignored: the counter is not writable from software. + if a.read { + b.put(a.rt, b.load(b.i64, at + offset_of!(CpuSystem, cntvct_el0))); + } + } + IccIar1El1 => { + // No group 1 interrupt is delivered by this model, so acknowledge reads spurious. + if a.read { + b.put(a.rt, b.k(b.i64, 1023)); + } + } + IccEoir1El1 => {} + Daif => { + let p = at + offset_of!(CpuSystem, daif); + if a.read { + b.put(a.rt, b.imm(b.i64, B::Shl, b.load(b.i64, p), 6)) + } else { + b.store( + p, + b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, b.reg(b.i64, a.rt, true), 6), + 0xf, + ), + ); + } + } + _ => { + if let Some(constant) = super::identification::value(a.register) { + b.put(a.rt, b.k(b.i64, constant)); + return; + } + let offset = + system_offset(a.register).expect("decoded system register has storage"); + let p = at + offset; + if a.read { + let v = if a.register == CntvCtlEl0 { + let counter = b.load(b.i64, at + offset_of!(CpuSystem, cntvct_el0)); + let compare = b.load(b.i64, at + offset_of!(CpuSystem, cntv_cval_el0)); + let reached = b.cmp(C::Uge, counter, compare); + b.bin( + b.i64, + B::BitOr, + b.load(b.i64, p), + b.sel(b.i64, reached, b.k(b.i64, 4), b.k(b.i64, 0)), + ) + } else { + b.load(b.i64, p) + }; + b.put(a.rt, v); + } else { + let from = b.reg(b.i64, a.rt, true); + b.store( + p, + if a.register == CntvCtlEl0 { + b.imm(b.i64, B::BitAnd, from, 3) + } else { + from + }, + ); + } + } + } + } +} +fn system_offset(r: SystemRegister) -> Option { + use SystemRegister::*; + Some(match r { + SpsrEl1 => offset_of!(CpuSystem, spsr_el1), + ElrEl1 => offset_of!(CpuSystem, elr_el1), + EsrEl1 => offset_of!(CpuSystem, esr_el1), + FarEl1 => offset_of!(CpuSystem, far_el1), + ParEl1 => offset_of!(CpuSystem, par_el1), + IsrEl1 => offset_of!(CpuSystem, isr_el1), + IccSreEl1 => offset_of!(CpuSystem, icc_sre_el1), + IccPmrEl1 => offset_of!(CpuSystem, icc_pmr_el1), + IccBpr0El1 => offset_of!(CpuSystem, icc_bpr0_el1), + IccCtlrEl1 => offset_of!(CpuSystem, icc_ctlr_el1), + IccIgrpen0El1 => offset_of!(CpuSystem, icc_igrpen0_el1), + IccIgrpen1El1 => offset_of!(CpuSystem, icc_igrpen1_el1), + IccBpr1El1 => offset_of!(CpuSystem, icc_bpr1_el1), + CntpCtlEl0 => offset_of!(CpuSystem, cntp_ctl_el0), + CntpCvalEl0 => offset_of!(CpuSystem, cntp_cval_el0), + ActlrEl1 => offset_of!(CpuSystem, actlr_el1), + VbarEl1 => offset_of!(CpuSystem, vbar_el1), + SctlrEl2 => offset_of!(CpuSystem, sctlr_el2), + TcrEl2 => offset_of!(CpuSystem, tcr_el2), + Ttbr0El2 => offset_of!(CpuSystem, ttbr0_el2), + Ttbr1El2 => offset_of!(CpuSystem, ttbr1_el2), + MairEl2 => offset_of!(CpuSystem, mair_el2), + VbarEl2 => offset_of!(CpuSystem, vbar_el2), + ElrEl2 => offset_of!(CpuSystem, elr_el2), + SpsrEl2 => offset_of!(CpuSystem, spsr_el2), + EsrEl2 => offset_of!(CpuSystem, esr_el2), + FarEl2 => offset_of!(CpuSystem, far_el2), + HcrEl2 => offset_of!(CpuSystem, hcr_el2), + TpidrEl2 => offset_of!(CpuSystem, tpidr_el2), + CntvoffEl2 => offset_of!(CpuSystem, cntvoff_el2), + CpacrEl1 => offset_of!(CpuSystem, cpacr_el1), + SctlrEl1 => offset_of!(CpuSystem, sctlr_el1), + Ttbr0El1 => offset_of!(CpuSystem, ttbr0_el1), + Ttbr1El1 => offset_of!(CpuSystem, ttbr1_el1), + TcrEl1 => offset_of!(CpuSystem, tcr_el1), + MairEl1 => offset_of!(CpuSystem, mair_el1), + TpidrEl0 => offset_of!(CpuSystem, tpidr_el0), + TpidrroEl0 => offset_of!(CpuSystem, tpidrro_el0), + CntvctEl0 => offset_of!(CpuSystem, cntvct_el0), + CntvCtlEl0 => offset_of!(CpuSystem, cntv_ctl_el0), + CntvCvalEl0 => offset_of!(CpuSystem, cntv_cval_el0), + TpidrEl1 => offset_of!(CpuSystem, tpidr_el1), + ContextidrEl1 => offset_of!(CpuSystem, contextidr_el1), + CsselrEl1 => offset_of!(CpuSystem, csselr_el1), + MdscrEl1 => offset_of!(CpuSystem, mdscr_el1), + CntkctlEl1 => offset_of!(CpuSystem, cntkctl_el1), + OslarEl1 => offset_of!(CpuSystem, oslar_el1), + OsdlrEl1 => offset_of!(CpuSystem, osdlr_el1), + Fpcr => offset_of!(CpuSystem, fpcr), + Fpsr => offset_of!(CpuSystem, fpsr), + Daif => offset_of!(CpuSystem, daif), + Apple(slot) => offset_of!(CpuSystem, apple) + 8 * usize::from(slot), + // After `system` in `Cpu`: relative to it, like every other register here. + AppleBank(slot) => { + offset_of!(Cpu, apple_bank) - offset_of!(Cpu, system) + 8 * usize::from(slot) + } + MidrEl1 => offset_of!(CpuSystem, midr_el1), + IdAa64pfr0El1 => offset_of!(CpuSystem, id_aa64pfr0_el1), + IdAa64mmfr0El1 => offset_of!(CpuSystem, id_aa64mmfr0_el1), + PacKey(slot) => offset_of!(CpuSystem, pac_keys) + 8 * usize::from(slot), + IdAa64isar1El1 => offset_of!(CpuSystem, id_aa64isar1_el1), + _ => return None, + }) +} + +/// EL1 registers that `HCR_EL2.E2H` redirects to their EL2 banks when accessed from EL2: +/// (EL1 field offset, EL2 field offset). +fn vhe_banks(r: SystemRegister) -> Option<(usize, usize)> { + use SystemRegister::*; + Some(match r { + SctlrEl1 => ( + offset_of!(CpuSystem, sctlr_el1), + offset_of!(CpuSystem, sctlr_el2), + ), + Ttbr0El1 => ( + offset_of!(CpuSystem, ttbr0_el1), + offset_of!(CpuSystem, ttbr0_el2), + ), + Ttbr1El1 => ( + offset_of!(CpuSystem, ttbr1_el1), + offset_of!(CpuSystem, ttbr1_el2), + ), + TcrEl1 => ( + offset_of!(CpuSystem, tcr_el1), + offset_of!(CpuSystem, tcr_el2), + ), + MairEl1 => ( + offset_of!(CpuSystem, mair_el1), + offset_of!(CpuSystem, mair_el2), + ), + VbarEl1 => ( + offset_of!(CpuSystem, vbar_el1), + offset_of!(CpuSystem, vbar_el2), + ), + ElrEl1 => ( + offset_of!(CpuSystem, elr_el1), + offset_of!(CpuSystem, elr_el2), + ), + SpsrEl1 => ( + offset_of!(CpuSystem, spsr_el1), + offset_of!(CpuSystem, spsr_el2), + ), + EsrEl1 => ( + offset_of!(CpuSystem, esr_el1), + offset_of!(CpuSystem, esr_el2), + ), + FarEl1 => ( + offset_of!(CpuSystem, far_el1), + offset_of!(CpuSystem, far_el2), + ), + _ => return None, + }) +} +impl Environment for CpuEnvironment { + fn x_offset(&self, r: u8) -> usize { + offset_of!(Cpu, x) + r as usize * 8 + } + fn sp_offset(&self) -> usize { + offset_of!(Cpu, sp) + } + fn flags_offset(&self) -> usize { + offset_of!(Cpu, flags) + } + fn v_offset(&self, r: u8) -> usize { + offset_of!(Cpu, v) + r as usize * 16 + } + fn begin_instruction(&mut self, _: &Builder) { + self.pending += 1; + } + fn exit(&mut self, b: &Builder, next_pc: Value) { + self.leave(b, next_pc) + } + fn memory(&mut self, b: &Builder, pc: u64, access: MemoryAccess<'_>) -> Result<(), LiftError> { + let MemoryAccess { + insn, + address, + writeback, + } = access; + match (*insn, address) { + (Instruction::Memory(a), Some(address)) => self.scalar(b, pc, a, address, writeback), + (Instruction::Literal(a), Some(address)) => self.literal(b, pc, a, address), + (Instruction::Pair(a), Some(address)) => self.pair(b, pc, a, address, writeback), + (Instruction::SimdLiteral(a), Some(address)) => { + self.simd_trap_access(b, address, a.bytes as u64, a.rt, None, None, pc, Trap::Load) + } + (Instruction::SimdPair(a), Some(address)) => { + self.simd_pair(b, pc, a, address, writeback) + } + (Instruction::SimdMemory(a), Some(address)) => { + self.simd_memory(b, pc, a, address, writeback) + } + (Instruction::Exclusive(a), _) => self.exclusive(b, pc, a), + (Instruction::Atomic(a), _) => self.atomic(b, pc, a), + (Instruction::SimdStruct(a), _) => self.simd_struct(b, pc, a), + _ => return Err(LiftError::Unsupported), + } + Ok(()) + } + fn event(&mut self, b: &Builder, pc: u64, insn: &Instruction) -> Result<(), LiftError> { + match *insn { + Instruction::System(a) => self.lower_system(b, a), + Instruction::Clrex(_) => b.byte(offset_of!(Cpu, monitor_valid), 0), + Instruction::Sev | Instruction::Sevl => b.byte(offset_of!(Cpu, event_set), 1), + // Cache maintenance by address has no effect on this machine. + Instruction::CacheOp(_) => {} + Instruction::AddressTranslate(a) => { + b.store(offset_of!(Cpu, address), b.reg(b.i64, a.rt, true)); + b.store( + offset_of!(Cpu, value), + b.k(b.i64, u64::from(a.write) | u64::from(a.user) << 1), + ); + self.trap(b, pc.wrapping_add(4), Trap::AddressTranslate); + } + Instruction::Wfe => self.trap(b, pc.wrapping_add(4), Trap::Wfe), + Instruction::Brk(imm) => { + b.store(offset_of!(Cpu, address), b.k(b.i64, u64::from(imm))); + self.trap(b, pc, Trap::Brk) + } + Instruction::Isb(_) => self.trap(b, pc.wrapping_add(4), Trap::Isb), + Instruction::Tlbi(_) => self.trap(b, pc.wrapping_add(4), Trap::Tlbi), + Instruction::DcZva(rt) => { + b.store(offset_of!(Cpu, address), b.reg(b.i64, rt, true)); + self.trap(b, pc.wrapping_add(4), Trap::DcZva) + } + Instruction::Eret => self.trap(b, pc.wrapping_add(4), Trap::Eret), + Instruction::Svc(imm) => { + b.store(offset_of!(Cpu, address), b.k(b.i64, u64::from(imm))); + self.trap(b, pc.wrapping_add(4), Trap::Svc) + } + Instruction::Psci => self.trap(b, pc.wrapping_add(4), Trap::Psci), + Instruction::Wfi => self.trap(b, pc.wrapping_add(4), Trap::Wfi), + _ => return Err(LiftError::Unsupported), + } + Ok(()) + } +} diff --git a/src/emulator/src/aarch64/mod.rs b/src/emulator/src/aarch64/mod.rs index 407a3b3..053b6fd 100644 --- a/src/emulator/src/aarch64/mod.rs +++ b/src/emulator/src/aarch64/mod.rs @@ -2,6 +2,7 @@ pub mod ahead; pub mod cache; pub mod compile; pub mod cpu; +mod environment; pub mod exception; pub mod fp; pub mod gxf; diff --git a/src/lift/aarch64/Cargo.toml b/src/lift/aarch64/Cargo.toml new file mode 100644 index 0000000..90e4cf8 --- /dev/null +++ b/src/lift/aarch64/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "volt-lift-aarch64" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +path = "src/lib.rs" + +[dependencies] +volt-ir.workspace = true +volt-target = { path = "../../target", default-features = false, features = ["aarch64"] } + +[lints] +workspace = true diff --git a/src/lift/aarch64/src/builder.rs b/src/lift/aarch64/src/builder.rs new file mode 100644 index 0000000..466b07d --- /dev/null +++ b/src/lift/aarch64/src/builder.rs @@ -0,0 +1,284 @@ +use crate::{Environment, Writeback}; +use core::cell::{Cell, RefCell}; +use volt_ir::{ + function::{self as ir, BinOp as B, CmpOp as C, Function, Opcode, Value}, + types::{IntDesc, Type, TypeKind}, +}; +use volt_target::aarch64::decode::{Addressing, Width}; + +/// Emits IR for one lifted function and knows where guest state lives. +/// +/// The builder owns the [`Function`] under construction, the block currently +/// being filled, the interned scalar types, and the state-pointer parameter +/// (`fn(state: *mut u8) -> u64`). Guest registers are addressed through the +/// offsets the [`Environment`] reported when the builder was created. +/// +/// Every method takes `&self`: emission order is the call order, and +/// [`Builder::set_block`] redirects subsequent emission, which is how an +/// environment builds fast and slow memory paths. +pub struct Builder { + f: RefCell, + /// Block currently being emitted into. + block: Cell, + state: Value, + /// 64-bit integer. + pub i64: Type, + /// 32-bit integer. + pub i32: Type, + /// 8-bit integer. + pub i8: Type, + /// One-bit comparison result. + pub boolean: Type, + /// Pointer; the type of the state pointer and of host addresses. + pub ptr: Type, + x: [usize; 31], + sp: usize, + flags: usize, + v: [usize; 32], +} +impl Builder { + pub(crate) fn new(env: &E) -> Self { + let mut f = Function::new(); + // A typical block is a few blocks and about a hundred instructions. + f.reserve(8, 128, 128); + let i64 = f.types.intern(TypeKind::Int(IntDesc { bits: 64 })); + let i32 = f.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let i8 = f.types.intern(TypeKind::Int(IntDesc { bits: 8 })); + let boolean = f.types.intern(TypeKind::Bool); + let ptr = f.types.ptr_global(); + let block = f.append_block(); + let state = f.append_block_param(block, ptr); + Self { + f: RefCell::new(f), + block: Cell::new(block), + state, + i64, + i32, + i8, + boolean, + ptr, + x: core::array::from_fn(|r| env.x_offset(r as u8)), + sp: env.sp_offset(), + flags: env.flags_offset(), + v: core::array::from_fn(|r| env.v_offset(r as u8)), + } + } + pub(crate) fn into_function(self) -> Function { + self.f.into_inner() + } + + // ---- guest state layout ---- + + /// The state pointer parameter of the function being built. + pub fn state(&self) -> Value { + self.state + } + /// Offset of general register `r`: `X0..=X30` for `r < 31`, `SP` for `r == 31`. + pub fn reg_at(&self, r: u8) -> usize { + if r == 31 { self.sp } else { self.x[r as usize] } + } + /// Offset of the 32-bit NZCV word. + pub fn flags_at(&self) -> usize { + self.flags + } + /// Offset of the low doubleword of vector register `r`. + pub fn v_at(&self, r: u8) -> usize { + self.v[r as usize] + } + + // ---- emission ---- + + /// Append `op` producing a value of type `t` to the current block. + pub fn emit(&self, t: Type, op: Opcode) -> Value { + self.f.borrow_mut().append_inst(self.block.get(), t, op) + } + /// An integer constant of type `t`. + pub fn k(&self, t: Type, n: u64) -> Value { + self.emit(t, Opcode::Iconst(n as i64)) + } + pub fn bin(&self, t: Type, op: B, a: Value, b: Value) -> Value { + self.emit(t, Opcode::Arith(ir::Arith { op, lhs: a, rhs: b })) + } + pub fn imm(&self, t: Type, op: B, a: Value, n: u64) -> Value { + // `ArithImm` instead of `Arith` over an `Iconst`: the backend encodes + // small immediates directly and `addrfold` only folds `ArithImm` adds + // into load/store addressing. + self.emit( + t, + Opcode::ArithImm(ir::ArithImm { + op, + lhs: a, + imm: n as i64, + }), + ) + } + /// Zero-extend or truncate `v` to `t`; `v` unchanged when it already has that type. + pub fn cv(&self, t: Type, v: Value) -> Value { + let from = self.f.borrow().value_type(v); + if from == t { + return v; + } + let kind = { + let f = self.f.borrow(); + let bits = |ty| match f.types.type_kind(ty) { + TypeKind::Int(d) => d.bits, + _ => 1, + }; + if bits(t) > bits(from) { + ir::ConvertKind::Zext + } else { + ir::ConvertKind::Trunc + } + }; + self.emit(t, Opcode::Convert(ir::Convert { value: v, kind })) + } + pub fn cmp(&self, op: C, a: Value, b: Value) -> Value { + self.emit( + self.boolean, + Opcode::Icmp(ir::Compare { op, lhs: a, rhs: b }), + ) + } + pub fn sel(&self, t: Type, c: Value, a: Value, b: Value) -> Value { + self.emit( + t, + Opcode::Select(ir::Select { + cond: c, + then: a, + else_: b, + }), + ) + } + /// The interned integer type of `bits` bits. + pub fn int(&self, bits: u16) -> Type { + self.f + .borrow_mut() + .types + .intern(TypeKind::Int(IntDesc { bits })) + } + pub(crate) fn ty(&self, w: Width) -> Type { + if w == Width::X64 { self.i64 } else { self.i32 } + } + + // ---- guest state access ---- + + /// Pointer to `state + at`. + pub fn addr(&self, at: usize) -> Value { + self.imm(self.ptr, B::Add, self.state, at as u64) + } + pub fn load_ptr(&self, t: Type, p: Value) -> Value { + self.emit( + t, + Opcode::Load(ir::Load { + ptr: p, + mem: ir::MemFlags::default(), + }), + ) + } + /// Load a `t` from `state + at`. + pub fn load(&self, t: Type, at: usize) -> Value { + self.load_ptr(t, self.addr(at)) + } + pub fn store_ptr(&self, p: Value, v: Value) { + self.f.borrow_mut().append_store(self.block.get(), v, p) + } + /// Store `v` to `state + at`. + pub fn store(&self, at: usize, v: Value) { + self.store_ptr(self.addr(at), v) + } + /// Store the 8-bit constant `v` to `state + at`. + pub fn byte(&self, at: usize, v: u64) { + self.store(at, self.k(self.i8, v)) + } + /// Read register `r` as a `t`. With `zero`, register 31 is `XZR` and reads + /// as zero; otherwise it is `SP`. + pub fn reg(&self, t: Type, r: u8, zero: bool) -> Value { + if zero && r >= 31 { + self.k(t, 0) + } else { + self.cv(t, self.load(self.i64, self.reg_at(r))) + } + } + /// Write `X`; a write to register 31 (`XZR`) is dropped. + pub fn put(&self, r: u8, v: Value) { + if r != 31 { + self.put_sp(r, v) + } + } + /// Write `X`, or `SP` for register 31. + pub fn put_sp(&self, r: u8, v: Value) { + self.store(self.reg_at(r), self.cv(self.i64, v)) + } + + // ---- control flow ---- + + /// Block currently being emitted into. + pub fn block(&self) -> ir::Block { + self.block.get() + } + /// Direct subsequent emission into `block`. + pub fn set_block(&self, block: ir::Block) { + self.block.set(block) + } + pub fn new_block(&self) -> ir::Block { + self.f.borrow_mut().append_block() + } + /// End the current block with a conditional branch. + pub fn branch_if(&self, c: Value, then: ir::Block, otherwise: ir::Block) { + self.f.borrow_mut().append_if( + self.block.get(), + c, + ir::EdgeDesc::bare(then), + ir::EdgeDesc::bare(otherwise), + ) + } + /// End the current block by returning `next_pc` from the function. An + /// environment's [`Environment::exit`] ends here; guest code never calls + /// this directly. + pub fn set_ret(&self, next_pc: Value) { + self.f + .borrow_mut() + .set_terminator(self.block.get(), ir::Terminator::Ret(ir::Ret::one(next_pc))) + } + + // ---- addressing ---- + + /// Effective address of a load/store with base `rn`, plus the base-register + /// update of a pre- or post-index mode. + /// + /// The update is returned instead of applied so each path can apply it its + /// own way once the access has succeeded: an access that faults must leave + /// the base register untouched so the retry sees it. + pub fn address(&self, rn: u8, a: Addressing) -> (Value, Option) { + let base = self.reg(self.i64, rn, false); + match a { + Addressing::Offset(n) => (self.imm(self.i64, B::Add, base, n as u64), None), + Addressing::PreIndex(n) => { + let address = self.imm(self.i64, B::Add, base, n as u64); + let writeback = Writeback { + register: rn, + value: address, + pre_index: true, + }; + (address, Some(writeback)) + } + Addressing::PostIndex(n) => { + let writeback = Writeback { + register: rn, + value: self.imm(self.i64, B::Add, base, n as u64), + pre_index: false, + }; + (base, Some(writeback)) + } + Addressing::Register { rm, extend, amount } => { + let v = self.extend(self.reg(self.i64, rm, true), extend); + let address = self.bin( + self.i64, + B::Add, + base, + self.imm(self.i64, B::Shl, v, amount as u64), + ); + (address, None) + } + } + } +} diff --git a/src/lift/aarch64/src/int.rs b/src/lift/aarch64/src/int.rs new file mode 100644 index 0000000..333ffe0 --- /dev/null +++ b/src/lift/aarch64/src/int.rs @@ -0,0 +1,383 @@ +//! Integer semantics: flags, conditions, shifts, extends, and the integer +//! instruction families too large to sit inline in the dispatcher. +use crate::Builder; +use volt_ir::{ + function::{BinOp as B, CmpOp as C, Value}, + types::Type, +}; +use volt_target::aarch64::decode::*; + +impl Builder { + pub(crate) fn not(&self, v: Value) -> Value { + self.imm(self.boolean, B::BitXor, v, 1) + } + pub(crate) fn flag(&self, flags: Value, bit: u32) -> Value { + self.cmp( + C::Ne, + self.imm(self.i32, B::BitAnd, flags, 1 << bit), + self.k(self.i32, 0), + ) + } + pub(crate) fn condition(&self, c: Condition) -> Value { + let f = self.load(self.i32, self.flags_at()); + let n = self.flag(f, 31); + let z = self.flag(f, 30); + let carry = self.flag(f, 29); + let v = self.flag(f, 28); + let changed = self.bin(self.boolean, B::BitXor, n, v); + let ordered = self.not(changed); + let positive = self.not(z); + match c { + Condition::Eq => z, + Condition::Ne => positive, + Condition::Cs => carry, + Condition::Cc => self.not(carry), + Condition::Mi => n, + Condition::Pl => self.not(n), + Condition::Vs => v, + Condition::Vc => self.not(v), + Condition::Hi => self.bin(self.boolean, B::BitAnd, carry, positive), + Condition::Ls => self.not(self.bin(self.boolean, B::BitAnd, carry, positive)), + Condition::Ge => ordered, + Condition::Lt => changed, + Condition::Gt => self.bin(self.boolean, B::BitAnd, positive, ordered), + Condition::Le => self.not(self.bin(self.boolean, B::BitAnd, positive, ordered)), + Condition::Al | Condition::Nv => self.k(self.boolean, 1), + } + } + pub(crate) fn pack(&self, n: Value, z: Value, c: Value, v: Value) -> Value { + let mut out = self.k(self.i32, 0); + for (bit, shift) in [(n, 31), (z, 30), (c, 29), (v, 28)] { + out = self.bin( + self.i32, + B::BitOr, + out, + self.imm(self.i32, B::Shl, self.cv(self.i32, bit), shift), + ); + } + out + } + pub(crate) fn flags( + &self, + w: Width, + op: ArithOp, + a: Value, + b: Value, + r: Value, + carry: Option, + ) -> Value { + let t = self.ty(w); + let zero = self.k(t, 0); + let n = self.cmp( + C::Ne, + self.imm(t, B::BitAnd, r, 1u64 << ((w as u32) - 1)), + zero, + ); + let z = self.cmp(C::Eq, r, zero); + let c = carry.unwrap_or_else(|| { + if op == ArithOp::Sub { + self.cmp(C::Uge, a, b) + } else { + self.cmp(C::Ult, r, a) + } + }); + let an = self.cmp(C::Slt, a, zero); + let bn = self.cmp(C::Slt, b, zero); + let rn = self.cmp(C::Slt, r, zero); + let diff = self.bin(self.boolean, B::BitXor, an, bn); + let flip = self.bin(self.boolean, B::BitXor, an, rn); + let v = self.bin( + self.boolean, + B::BitAnd, + if op == ArithOp::Sub { + diff + } else { + self.not(diff) + }, + flip, + ); + self.pack(n, z, c, v) + } + pub(crate) fn logic_flags(&self, w: Width, r: Value) { + let t = self.ty(w); + let zero = self.k(t, 0); + let n = self.cmp( + C::Ne, + self.imm(t, B::BitAnd, r, 1u64 << ((w as u32) - 1)), + zero, + ); + let z = self.cmp(C::Eq, r, zero); + let no = self.k(self.boolean, 0); + self.store(self.flags_at(), self.pack(n, z, no, no)); + } + pub fn shift(&self, w: Width, v: Value, s: Shift, n: u64) -> Value { + if n == 0 { + return v; + } + let t = self.ty(w); + match s { + Shift::Lsl => self.imm(t, B::Shl, v, n), + Shift::Lsr => self.imm(t, B::Shr, v, n), + Shift::Asr => self.imm(t, B::Sar, v, n), + Shift::Ror => self.bin( + t, + B::BitOr, + self.imm(t, B::Shr, v, n), + self.imm(t, B::Shl, v, w as u64 - n), + ), + } + } + pub fn extend(&self, v: Value, e: Extend) -> Value { + match e { + Extend::Uxtx | Extend::Sxtx => v, + Extend::Uxtb => self.imm(self.i64, B::BitAnd, v, 0xff), + Extend::Uxth => self.imm(self.i64, B::BitAnd, v, 0xffff), + Extend::Uxtw => self.imm(self.i64, B::BitAnd, v, 0xffffffff), + Extend::Sxtb | Extend::Sxth | Extend::Sxtw => { + let n = match e { + Extend::Sxtb => 56, + Extend::Sxth => 48, + _ => 32, + }; + let raised = self.imm(self.i64, B::Shl, v, n); + self.shift(Width::X64, raised, Shift::Asr, n) + } + } + } +} +pub(crate) fn arithmetic_op(op: ArithOp) -> B { + if op == ArithOp::Sub { B::Sub } else { B::Add } +} +pub(crate) fn logic_op(op: LogicOp) -> B { + match op { + LogicOp::BitAnd => B::BitAnd, + LogicOp::BitOr => B::BitOr, + LogicOp::BitXor => B::BitXor, + } +} +pub(crate) fn low_mask(n: u8) -> u64 { + if n >= 64 { u64::MAX } else { (1u64 << n) - 1 } +} + +pub(crate) fn variable(b: &Builder, a: Variable) { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let rhs = b.reg(t, a.rm, true); + let bits = a.width as u64; + let r = match a.op { + VariableOp::Lsl | VariableOp::Lsr | VariableOp::Asr | VariableOp::Ror => { + let distance = b.imm(t, B::BitAnd, rhs, bits - 1); + match a.op { + VariableOp::Lsl => b.bin(t, B::Shl, lhs, distance), + VariableOp::Lsr => b.bin(t, B::Shr, lhs, distance), + VariableOp::Asr => b.bin(t, B::Sar, lhs, distance), + VariableOp::Ror => { + let back = b.imm( + t, + B::BitAnd, + b.bin(t, B::Sub, b.k(t, bits), distance), + bits - 1, + ); + b.bin( + t, + B::BitOr, + b.bin(t, B::Shr, lhs, distance), + b.bin(t, B::Shl, lhs, back), + ) + } + _ => unreachable!(), + } + } + VariableOp::Udiv | VariableOp::Sdiv => { + let zero = b.k(t, 0); + let one = b.k(t, 1); + let by_zero = b.cmp(C::Eq, rhs, zero); + if a.op == VariableOp::Udiv { + let safe = b.sel(t, by_zero, one, rhs); + b.sel(t, by_zero, zero, b.bin(t, B::UDiv, lhs, safe)) + } else { + let minus_one = b.cmp(C::Eq, rhs, b.k(t, u64::MAX)); + let safe = b.sel(t, minus_one, one, b.sel(t, by_zero, one, rhs)); + let quotient = b.bin(t, B::SDiv, lhs, safe); + let negated = b.bin(t, B::Sub, zero, lhs); + b.sel(t, by_zero, zero, b.sel(t, minus_one, negated, quotient)) + } + } + }; + if a.rd != 31 { + b.put(a.rd, r) + } +} +pub(crate) fn bitfield(b: &Builder, a: Bitfield) { + let t = b.ty(a.width); + let bits = a.width as u64; + let len = a.len as u64; + let lsb = a.lsb as u64; + let source = b.reg(t, a.rn, true); + let mask = low_mask(a.len); + let width_mask = if a.width == Width::X64 { + u64::MAX + } else { + u32::MAX as u64 + }; + let extract = |signed| { + let raised = b.imm(t, B::Shl, source, bits - lsb - len); + b.shift( + a.width, + raised, + if signed { Shift::Asr } else { Shift::Lsr }, + bits - len, + ) + }; + let r = match a.op { + BitfieldOp::Unsigned => { + if a.extract { + extract(false) + } else { + b.imm( + t, + B::Shl, + b.imm(t, B::BitAnd, source, mask & width_mask), + lsb, + ) + } + } + BitfieldOp::Signed => { + if a.extract { + extract(true) + } else { + let raised = b.imm(t, B::Shl, source, bits - len); + let extended = b.shift(a.width, raised, Shift::Asr, bits - len); + b.imm(t, B::Shl, extended, lsb) + } + } + BitfieldOp::Insert => { + let value = if a.extract { + extract(false) + } else { + b.imm( + t, + B::Shl, + b.imm(t, B::BitAnd, source, mask & width_mask), + lsb, + ) + }; + let place = if a.extract { 0 } else { lsb }; + let hole = (mask << place) & width_mask; + let kept = b.imm(t, B::BitAnd, b.reg(t, a.rd, true), !hole & width_mask); + b.bin(t, B::BitOr, kept, value) + } + }; + if a.rd != 31 { + b.put(a.rd, r) + } +} +fn repeated(pattern: u64, unit: u32, bits: u32) -> u64 { + let mut r = 0; + let mut at = 0; + while at < bits { + r |= pattern << at; + at += unit + } + r +} +fn swap(b: &Builder, t: Type, v: Value, mask: u64, n: u64) -> Value { + let low = b.imm(t, B::BitAnd, v, mask); + let high = b.imm(t, B::BitAnd, b.imm(t, B::Shr, v, n), mask); + b.bin(t, B::BitOr, b.imm(t, B::Shl, low, n), high) +} +pub(crate) fn unary(b: &Builder, a: Unary) { + let t = b.ty(a.width); + let bits = a.width as u32; + let mut v = b.reg(t, a.rn, true); + match a.op { + UnaryOp::Rev16 => v = swap(b, t, v, repeated(0xff, 16, bits), 8), + UnaryOp::Rev32 => { + v = swap(b, t, v, repeated(0xff, 16, bits), 8); + v = swap(b, t, v, repeated(0xffff, 32, bits), 16); + } + UnaryOp::Rev | UnaryOp::Rbit => { + if a.op == UnaryOp::Rbit { + for (mask, unit, n) in [(1, 2, 1), (3, 4, 2), (15, 8, 4)] { + v = swap(b, t, v, repeated(mask, unit, bits), n) + } + } + v = swap(b, t, v, repeated(0xff, 16, bits), 8); + v = swap(b, t, v, repeated(0xffff, 32, bits), 16); + if bits == 64 { + v = b.bin( + t, + B::BitOr, + b.imm(t, B::Shl, v, 32), + b.imm(t, B::Shr, v, 32), + ) + } + } + UnaryOp::Clz => { + let original = v; + let mut count = b.k(t, 0); + let mut step = bits / 2; + while step >= 1 { + let top = b.imm(t, B::Shr, v, (bits - step) as u64); + let empty = b.cmp(C::Eq, top, b.k(t, 0)); + v = b.sel(t, empty, b.imm(t, B::Shl, v, step as u64), v); + count = b.sel(t, empty, b.imm(t, B::Add, count, step as u64), count); + step /= 2; + } + v = b.sel( + t, + b.cmp(C::Eq, original, b.k(t, 0)), + b.k(t, bits as u64), + count, + ); + } + } + if a.rd != 31 { + b.put(a.rd, v) + } +} +pub(crate) fn high(b: &Builder, a: MultiplyHigh) { + let t = b.i64; + let x = b.reg(t, a.rn, true); + let y = b.reg(t, a.rm, true); + let xl = b.imm(t, B::BitAnd, x, 0xffffffff); + let yl = b.imm(t, B::BitAnd, y, 0xffffffff); + let xh = b.imm(t, B::Shr, x, 32); + let yh = b.imm(t, B::Shr, y, 32); + let ll = b.bin(t, B::Mul, xl, yl); + let lh = b.bin(t, B::Mul, xl, yh); + let hl = b.bin(t, B::Mul, xh, yl); + let hh = b.bin(t, B::Mul, xh, yh); + let middle = b.bin( + t, + B::Add, + b.bin( + t, + B::Add, + b.imm(t, B::Shr, ll, 32), + b.imm(t, B::BitAnd, lh, 0xffffffff), + ), + b.imm(t, B::BitAnd, hl, 0xffffffff), + ); + let mut high = b.bin( + t, + B::Add, + b.bin(t, B::Add, hh, b.imm(t, B::Shr, lh, 32)), + b.bin( + t, + B::Add, + b.imm(t, B::Shr, hl, 32), + b.imm(t, B::Shr, middle, 32), + ), + ); + if a.signed { + let zero = b.k(t, 0); + let xn = b.bin(t, B::Sub, zero, b.imm(t, B::Shr, x, 63)); + let yn = b.bin(t, B::Sub, zero, b.imm(t, B::Shr, y, 63)); + high = b.bin(t, B::Sub, high, b.bin(t, B::BitAnd, y, xn)); + high = b.bin(t, B::Sub, high, b.bin(t, B::BitAnd, x, yn)); + } + if a.rd != 31 { + b.put(a.rd, high) + } +} diff --git a/src/lift/aarch64/src/lib.rs b/src/lift/aarch64/src/lib.rs new file mode 100644 index 0000000..000962e --- /dev/null +++ b/src/lift/aarch64/src/lib.rs @@ -0,0 +1,516 @@ +//! AArch64 instruction semantics lifted to [`volt_ir`], independent of any +//! particular machine. +//! +//! [`Lifter`] turns decoded [`Instruction`]s into IR over *guest state held in +//! memory*. Every lifted function has the signature +//! +//! ```text +//! fn(state: *mut u8) -> u64 +//! ``` +//! +//! It reads and writes guest registers at `state + offset`, and **returns the +//! next guest pc**. Which offsets, and everything that is not pure +//! architecture, belongs to the [`Environment`] the lifter is generic over: +//! +//! * **Layout.** [`Environment::x_offset`], [`sp_offset`](Environment::sp_offset), +//! [`flags_offset`](Environment::flags_offset) and +//! [`v_offset`](Environment::v_offset) say where `X0..=X30`, `SP`, the NZCV +//! word (`u32`, flags in bits 31..28) and the 128-bit vector registers (two +//! little-endian `u64`s) live. They are read once, when the [`Lifter`] is +//! created. +//! * **Memory.** Loads, stores, pairs, literals, exclusives, atomics and SIMD +//! structure accesses are handed to [`Environment::memory`] with the +//! effective address already computed. The environment decides how guest +//! memory is reached: a plain host load, a TLB probe, a call out, a trap. +//! * **Events.** System registers, `SVC`, `BRK`, `WFI`/`WFE`/`SEV`, barriers +//! that need machine action, TLB maintenance, `ERET`, `DC ZVA`, `AT`, +//! `CLREX` and cache operations go to [`Environment::event`]. +//! * **Exits.** Every path that leaves the function (branches, fall-through +//! at the end of a block, anything the environment ends itself) goes through +//! [`Environment::exit`], which by default returns the next pc. +//! +//! Everything else (integer ALU, flags, shifts, bitfields, multiplies, +//! conditional select/compare, `ADR`, branches and the register-only SIMD +//! families) is architecture and is lifted here. +//! +//! The lifter lifts instructions; the *caller* owns the block: it decodes, +//! decides where a block ends (see [`Instruction::terminates_with`]), and +//! calls [`Lifter::exit`] when a block runs off its end without a +//! terminating instruction. An instruction that falls through leaves the +//! [`Builder`] positioned for the next one. An instruction that does not (a +//! branch, or a trap the environment ends with [`Environment::exit`]) has +//! already terminated the current block. An environment must therefore end the +//! block for every forwarded instruction for which `terminates_with` is true. + +#![no_std] +extern crate alloc; +#[cfg(test)] +extern crate std; + +mod builder; +mod int; +mod simd; +#[cfg(test)] +mod tests; + +pub use builder::Builder; +use core::fmt; +use volt_ir::function::{BinOp as B, CmpOp as C, Function, Value}; +use volt_target::aarch64::decode::*; + +/// A load, store or other guest memory instruction handed to +/// [`Environment::memory`]. +#[derive(Clone, Copy)] +pub struct MemoryAccess<'a> { + /// The decoded instruction: `Memory`, `Pair`, `Literal`, `SimdLiteral`, + /// `SimdMemory`, `SimdPair`, `Exclusive`, `Atomic` or `SimdStruct`. + pub insn: &'a Instruction, + /// The effective address, for the forms whose address is computed from the + /// addressing mode: `Memory`, `Pair`, `SimdMemory`, `SimdPair` (the base + /// for post-indexing, the updated base for pre-indexing), `Literal` and + /// `SimdLiteral` (a constant). `None` for `Exclusive`, `Atomic` and + /// `SimdStruct`, which address through a plain base register the + /// environment reads itself. + pub address: Option, + /// Base-register update of a pre- or post-index mode. The lifter does not + /// apply it: an access that can fault must leave the base untouched until + /// it has succeeded, so the environment writes it (with + /// [`Builder::put_sp`]) once it knows how. + pub writeback: Option, +} + +/// The base-register update of a pre- or post-indexed access. +#[derive(Clone, Copy, Debug)] +pub struct Writeback { + /// The base register; 31 is `SP`. + pub register: u8, + /// Its new value. + pub value: Value, + /// Pre-indexed: the access address is `value` itself. Otherwise the access + /// uses the old base and `value` is computed after it. + pub pre_index: bool, +} + +/// Why an instruction could not be lifted. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LiftError { + /// The [`Environment`] does not implement this instruction. + Unsupported, +} +impl fmt::Display for LiftError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Unsupported => f.write_str("instruction is not supported by the environment"), + } + } +} +impl core::error::Error for LiftError {} + +/// What a machine provides to the lifter: where guest state lives, how memory +/// is reached, and what instructions with machine effects do. See the +/// [crate documentation](crate). +pub trait Environment { + // ---- guest state layout (byte offsets from the state pointer) ---- + + /// Offset of `X` (`u64`), `r` in `0..=30`. + fn x_offset(&self, r: u8) -> usize; + /// Offset of the stack pointer (`u64`). + fn sp_offset(&self) -> usize; + /// Offset of the NZCV word (`u32`), flags in bits 31..28. + fn flags_offset(&self) -> usize; + /// Offset of vector register `V` (16 bytes: two little-endian `u64`s), `r` in `0..=31`. + fn v_offset(&self, r: u8) -> usize; + + // ---- hooks ---- + + /// Called once per guest instruction before it is lifted. An emulator + /// counts retired instructions here. + fn begin_instruction(&mut self, b: &Builder) { + let _ = b; + } + /// Called on every path that leaves the function with the next guest pc. + /// The default ends the current block with `Ret(next_pc)`; an environment + /// overrides it to do work on every exit first (for example flushing a + /// pending cycle counter) and then calls [`Builder::set_ret`]. + fn exit(&mut self, b: &Builder, next_pc: Value) { + b.set_ret(next_pc) + } + /// A load, store, exclusive, atomic or SIMD structure access at guest `pc`. + /// The environment emits whatever reaches memory; see [`MemoryAccess`] + /// for what the lifter has already computed. It must end the block (with + /// [`Self::exit`]) for instructions that terminate it. + fn memory(&mut self, b: &Builder, pc: u64, access: MemoryAccess<'_>) -> Result<(), LiftError>; + /// An instruction whose effect belongs to the machine: `System` + /// (`MRS`/`MSR`/`SYS`), `Svc`, `Brk`, `Wfi`, `Wfe`, `Sev`, `Sevl`, `Isb`, + /// `Tlbi`, `Eret`, `DcZva`, `AddressTranslate`, `Clrex`, `Psci` and + /// `CacheOp`. It receives the decoded instruction. + fn event(&mut self, b: &Builder, pc: u64, insn: &Instruction) -> Result<(), LiftError>; +} + +/// Lifts decoded AArch64 instructions into one [`Function`]. +pub struct Lifter { + b: Builder, + env: E, +} +impl Lifter { + /// Start a function with the layout `env` reports. + pub fn new(env: E) -> Self { + Self { + b: Builder::new(&env), + env, + } + } + /// The builder and the environment, for callers that emit IR of their own + /// between instructions (a block driver handling words it does not lift). + pub fn parts(&mut self) -> (&Builder, &mut E) { + (&self.b, &mut self.env) + } + /// The function ends at `next_pc`: the block fell off its end without a + /// terminating instruction. + pub fn exit(&mut self, next_pc: u64) { + let next = self.b.k(self.b.i64, next_pc); + self.env.exit(&self.b, next) + } + /// Finish: the function built so far, and the environment back. + pub fn into_parts(self) -> (Function, E) { + (self.b.into_function(), self.env) + } + /// Lift one decoded instruction at guest address `pc`. + pub fn lift(&mut self, pc: u64, insn: Instruction) -> Result<(), LiftError> { + let (b, env) = (&self.b, &mut self.env); + env.begin_instruction(b); + match insn { + Instruction::Movz(a) | Instruction::Movn(a) => { + let t = b.ty(a.width); + let bits = (a.immediate as u64) << a.shift; + let value = if matches!(insn, Instruction::Movn(_)) { + !bits + } else { + bits + }; + if a.rd != 31 { + b.put(a.rd, b.k(t, value)); + } + } + Instruction::Movk(a) => { + if a.rd != 31 { + let t = b.ty(a.width); + let old = b.reg(t, a.rd, false); + let kept = b.imm(t, B::BitAnd, old, !(0xffffu64 << a.shift)); + b.put( + a.rd, + b.imm(t, B::BitOr, kept, (a.immediate as u64) << a.shift), + ); + } + } + Instruction::ArithImm(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, false); + let rhs = b.k(t, a.immediate as u64); + let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); + if a.flags { + b.put(a.rd, r); + b.store(b.flags_at(), b.flags(a.width, a.op, lhs, rhs, r, None)); + } else { + b.put_sp(a.rd, r) + } + } + Instruction::ArithReg(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let rhs = b.shift(a.width, b.reg(t, a.rm, true), a.shift, a.amount as u64); + let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); + if a.rd != 31 { + b.put(a.rd, r) + } + if a.flags { + b.store(b.flags_at(), b.flags(a.width, a.op, lhs, rhs, r, None)); + } + } + Instruction::ArithExt(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, false); + let extended = b.extend(b.reg(b.i64, a.rm, true), a.extend); + let rhs = b.cv(t, b.imm(b.i64, B::Shl, extended, a.amount as u64)); + let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); + if a.flags { + b.put(a.rd, r); + b.store(b.flags_at(), b.flags(a.width, a.op, lhs, rhs, r, None)); + } else { + b.put_sp(a.rd, r) + } + } + Instruction::LogicReg(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let shifted = b.shift(a.width, b.reg(t, a.rm, true), a.shift, a.amount as u64); + let rhs = if a.invert { + b.imm(t, B::BitXor, shifted, u64::MAX) + } else { + shifted + }; + let r = b.bin(t, int::logic_op(a.op), lhs, rhs); + b.put(a.rd, r); + if a.flags { + b.logic_flags(a.width, r) + } + } + Instruction::LogicImm(a) => { + let t = b.ty(a.width); + let r = b.imm(t, int::logic_op(a.op), b.reg(t, a.rn, true), a.immediate); + if a.flags { + b.put(a.rd, r); + b.logic_flags(a.width, r) + } else { + b.put_sp(a.rd, r) + } + } + Instruction::ArithCarry(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let loaded = b.reg(t, a.rm, true); + let rhs = if a.op == ArithOp::Sub { + b.imm(t, B::BitXor, loaded, u64::MAX) + } else { + loaded + }; + let carry = b.cv( + t, + b.imm( + b.i32, + B::BitAnd, + b.imm(b.i32, B::Shr, b.load(b.i32, b.flags_at()), 29), + 1, + ), + ); + let partial = b.bin(t, B::Add, lhs, rhs); + let r = b.bin(t, B::Add, partial, carry); + if a.rd != 31 { + b.put(a.rd, r) + } + if a.flags { + let out = b.bin( + b.boolean, + B::BitOr, + b.cmp(C::Ult, partial, lhs), + b.cmp(C::Ult, r, partial), + ); + b.store( + b.flags_at(), + b.flags(a.width, ArithOp::Add, lhs, rhs, r, Some(out)), + ); + } + } + Instruction::CondCompare(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let rhs = match a.operand { + CompareOperand::Register(r) => b.reg(t, r, true), + CompareOperand::Immediate(n) => b.k(t, n as u64), + }; + let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); + let flags = b.flags(a.width, a.op, lhs, rhs, r, None); + b.store( + b.flags_at(), + b.sel( + b.i32, + b.condition(a.cond), + flags, + b.k(b.i32, (a.nzcv as u64) << 28), + ), + ); + } + Instruction::Variable(a) => int::variable(b, a), + Instruction::Bitfield(a) => int::bitfield(b, a), + Instruction::Unary(a) => int::unary(b, a), + Instruction::Extract(a) => { + let t = b.ty(a.width); + let low = b.reg(t, a.rm, true); + let r = if a.lsb == 0 { + low + } else { + b.bin( + t, + B::BitOr, + b.imm(t, B::Shr, low, a.lsb as u64), + b.imm( + t, + B::Shl, + b.reg(t, a.rn, true), + a.width as u64 - a.lsb as u64, + ), + ) + }; + if a.rd != 31 { + b.put(a.rd, r) + } + } + Instruction::Mul(a) => { + let t = b.ty(a.width); + let p = b.bin(t, B::Mul, b.reg(t, a.rn, true), b.reg(t, a.rm, true)); + let r = b.bin(t, int::arithmetic_op(a.op), b.reg(t, a.ra, true), p); + if a.rd != 31 { + b.put(a.rd, r) + } + } + Instruction::MulLong(a) => { + let e = if a.signed { Extend::Sxtw } else { Extend::Uxtw }; + let p = b.bin( + b.i64, + B::Mul, + b.extend(b.reg(b.i64, a.rn, true), e), + b.extend(b.reg(b.i64, a.rm, true), e), + ); + let r = b.bin(b.i64, int::arithmetic_op(a.op), b.reg(b.i64, a.ra, true), p); + if a.rd != 31 { + b.put(a.rd, r) + } + } + Instruction::MulHigh(a) => int::high(b, a), + Instruction::Adr(a) => { + let here = if a.page { pc & 0xfffffffffffff000 } else { pc }; + b.put(a.rd, b.k(b.i64, here.wrapping_add(a.offset as u64))); + } + Instruction::Csel(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let rhs = b.reg(t, a.rm, true); + let alt = match a.opc { + 0 => rhs, + 1 => b.imm(t, B::Add, rhs, 1), + 2 => b.imm(t, B::BitXor, rhs, u64::MAX), + 3 => b.bin(t, B::Sub, b.k(t, 0), rhs), + _ => unreachable!(), + }; + if a.rd != 31 { + b.put(a.rd, b.sel(t, b.condition(a.cond), lhs, alt)); + } + } + Instruction::TestBranch(a) => { + let t = b.ty(a.width); + let v = b.reg(t, a.rt, true); + let subject = if let Some(bit) = a.bit_index { + b.imm(t, B::BitAnd, v, 1u64 << bit) + } else { + v + }; + let taken = b.cmp(if a.negate { C::Ne } else { C::Eq }, subject, b.k(t, 0)); + branch(env, b, pc, a.offset, taken); + } + Instruction::BCond(a) => { + let taken = b.condition(a.cond); + branch(env, b, pc, a.offset, taken) + } + Instruction::B(offset) => { + let target = b.k(b.i64, pc.wrapping_add(offset as u64)); + env.exit(b, target) + } + Instruction::Call(a) => { + if a.link { + b.put(30, b.k(b.i64, pc.wrapping_add(4))) + } + let target = b.k(b.i64, pc.wrapping_add(a.target as u64)); + env.exit(b, target); + } + Instruction::Indirect(a) => { + let target = b.reg(b.i64, a.rn, false); + if a.link { + b.put(30, b.k(b.i64, pc.wrapping_add(4))) + } + env.exit(b, target); + } + + Instruction::Memory(a) => { + let (address, writeback) = b.address(a.rn, a.addressing); + memory(env, b, pc, &insn, Some(address), writeback)? + } + Instruction::Pair(a) => { + let (address, writeback) = b.address(a.rn, a.addressing); + memory(env, b, pc, &insn, Some(address), writeback)? + } + Instruction::SimdMemory(a) => { + let (address, writeback) = b.address(a.rn, a.addressing); + memory(env, b, pc, &insn, Some(address), writeback)? + } + Instruction::SimdPair(a) => { + let (address, writeback) = b.address(a.rn, a.addressing); + memory(env, b, pc, &insn, Some(address), writeback)? + } + Instruction::Literal(a) => { + let address = b.k(b.i64, pc.wrapping_add(a.offset as u64)); + memory(env, b, pc, &insn, Some(address), None)? + } + Instruction::SimdLiteral(a) => { + let address = b.k(b.i64, pc.wrapping_add(a.offset as u64)); + memory(env, b, pc, &insn, Some(address), None)? + } + Instruction::Exclusive(_) | Instruction::Atomic(_) | Instruction::SimdStruct(_) => { + memory(env, b, pc, &insn, None, None)? + } + Instruction::SimdImm(a) => simd::imm(b, a), + Instruction::SimdMove { rd, rn } => simd::mov(b, rd, rn), + Instruction::SimdInsert(a) => simd::insert(b, a), + Instruction::SimdInsertElement(a) => simd::insert_element(b, a), + Instruction::SimdExtract(a) => simd::extract(b, a), + Instruction::SimdDupElement(a) => simd::dup_element(b, a), + Instruction::SimdDup { rd, rn, esize, q } => simd::dup(b, rd, rn, esize, q), + Instruction::SimdExt(a) => simd::ext(b, a), + Instruction::SimdCompareZero(a) => simd::compare_zero(b, a), + Instruction::SimdTable(a) => simd::table(b, a), + Instruction::SimdFmov(a) => simd::fmov(b, a), + Instruction::SimdAlu(a) => simd::alu(b, a), + // Architecturally no-ops: nothing to emit. + Instruction::Nop + | Instruction::Hint(_) + | Instruction::Prefetch(_) + | Instruction::PrefetchLiteral(_) + | Instruction::Dsb(_) + | Instruction::Dmb(_) + | Instruction::Yield => {} + Instruction::System(_) + | Instruction::Svc(_) + | Instruction::Brk(_) + | Instruction::Wfi + | Instruction::Wfe + | Instruction::Sev + | Instruction::Sevl + | Instruction::Isb(_) + | Instruction::Tlbi(_) + | Instruction::Eret + | Instruction::DcZva(_) + | Instruction::AddressTranslate(_) + | Instruction::Clrex(_) + | Instruction::Psci + | Instruction::CacheOp(_) => env.event(b, pc, &insn)?, + } + Ok(()) + } +} +fn memory( + env: &mut E, + b: &Builder, + pc: u64, + insn: &Instruction, + address: Option, + writeback: Option, +) -> Result<(), LiftError> { + env.memory( + b, + pc, + MemoryAccess { + insn, + address, + writeback, + }, + ) +} +/// End the function at `pc + offset` when `taken`, otherwise at `pc + 4`. +fn branch(env: &mut E, b: &Builder, pc: u64, offset: i64, taken: Value) { + let next = b.sel( + b.i64, + taken, + b.k(b.i64, pc.wrapping_add(offset as u64)), + b.k(b.i64, pc.wrapping_add(4)), + ); + env.exit(b, next) +} diff --git a/src/lift/aarch64/src/simd.rs b/src/lift/aarch64/src/simd.rs new file mode 100644 index 0000000..a9d24c7 --- /dev/null +++ b/src/lift/aarch64/src/simd.rs @@ -0,0 +1,859 @@ +//! Register-only SIMD semantics: immediates, moves, lane insert/extract/dup, +//! `ext`, compare-against-zero, table lookup, `fmov` and the integer ALU. +//! +//! Every function works on the 128-bit vector registers in guest state as two +//! little-endian doublewords at [`Builder::v_at`]. +use crate::Builder; +use alloc::vec::Vec; +use volt_ir::function::{BinOp as B, CmpOp as C, Value}; +use volt_target::aarch64::decode::*; +pub(crate) fn imm(b: &Builder, a: SimdImmediate) { + let at = b.v_at(a.rd); + match a.combine { + ImmCombine::Set => { + b.store(at, b.k(b.i64, a.low)); + b.store(at + 8, b.k(b.i64, a.high)); + } + ImmCombine::Or | ImmCombine::AndNot => { + // A 64-bit form (`q == false`) also clears the upper half. + let (op, imm) = match a.combine { + ImmCombine::Or => (B::BitOr, a.low), + _ => (B::BitAnd, !a.low), + }; + b.store(at, b.imm(b.i64, op, b.load(b.i64, at), imm)); + b.store( + at + 8, + if a.q { + b.imm(b.i64, op, b.load(b.i64, at + 8), imm) + } else { + b.k(b.i64, 0) + }, + ); + } + } +} +pub(crate) fn mov(b: &Builder, rd: u8, rn: u8) { + let at_d = b.v_at(rd); + let at_n = b.v_at(rn); + let lo = b.load(b.i64, at_n); + let hi = b.load(b.i64, at_n + 8); + b.store(at_d, lo); + b.store(at_d + 8, hi); +} +pub(crate) fn insert(b: &Builder, a: SimdInsert) { + // Replace one lane of the destination with the low bits of a GPR. + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let at_d = b.v_at(a.rd); + let per_dw = (64 / ebits) as usize; + let word = a.index as usize / per_dw; + let shift = (a.index as usize % per_dw) as u64 * ebits; + let old = b.load(b.i64, at_d + word * 8); + let cleared = b.bin(b.i64, B::BitAnd, old, b.k(b.i64, !(emask << shift))); + let value = b.bin( + b.i64, + B::BitAnd, + b.reg(b.i64, a.rn, true), + b.k(b.i64, emask), + ); + b.store( + at_d + word * 8, + b.bin(b.i64, B::BitOr, cleared, b.imm(b.i64, B::Shl, value, shift)), + ); +} +pub(crate) fn insert_element(b: &Builder, a: SimdInsertElement) { + // Copy one lane of Vn over one lane of Vd. The source is read before + // the store, so `rd == rn` is fine. + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let per_dw = (64 / ebits) as usize; + let at_d = b.v_at(a.rd); + let at_n = b.v_at(a.rn); + let (src_word, src_shift) = ( + a.src as usize / per_dw, + (a.src as usize % per_dw) as u64 * ebits, + ); + let (dst_word, dst_shift) = ( + a.dst as usize / per_dw, + (a.dst as usize % per_dw) as u64 * ebits, + ); + let source = b.load(b.i64, at_n + src_word * 8); + let value = b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, source, src_shift), + emask, + ); + let old = b.load(b.i64, at_d + dst_word * 8); + let cleared = b.imm(b.i64, B::BitAnd, old, !(emask << dst_shift)); + b.store( + at_d + dst_word * 8, + b.bin( + b.i64, + B::BitOr, + cleared, + b.imm(b.i64, B::Shl, value, dst_shift), + ), + ); +} +pub(crate) fn extract(b: &Builder, a: SimdExtract) { + // Move one lane into the low bits of a GPR, zero-extended. + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let at_n = b.v_at(a.rn); + let per_dw = (64 / ebits) as usize; + let word = a.index as usize / per_dw; + let shift = (a.index as usize % per_dw) as u64 * ebits; + let source = b.load(b.i64, at_n + word * 8); + let value = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, source, shift), emask); + b.put(a.rd, value); +} +pub(crate) fn dup_element(b: &Builder, a: SimdDupElement) { + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let at_n = b.v_at(a.rn); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + // Locate the requested element across the 128-bit source. + let lane = a.index as usize; + let lanes_per_dw = (64 / ebits) as usize; + let src = if lane / lanes_per_dw == 0 { + vn_lo + } else { + vn_hi + }; + let shift = (lane % lanes_per_dw) as u64 * ebits; + let element = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, src, shift), emask); + // Replicate the element across the destination lanes. + let mut acc = b.k(b.i64, 0); + let lanes = (64 / ebits) as usize; + for lane in 0..lanes { + acc = b.bin( + b.i64, + B::BitOr, + acc, + b.imm(b.i64, B::Shl, element, lane as u64 * ebits), + ); + } + let at_d = b.v_at(a.rd); + b.store(at_d, acc); + b.store(at_d + 8, if a.q { acc } else { b.k(b.i64, 0) }); +} +pub(crate) fn dup(b: &Builder, rd: u8, rn: u8, esize: u8, q: bool) { + let r = b.reg(b.i64, rn, true); + let val64 = match esize { + 0 => { + let byte = b.imm(b.i64, B::BitAnd, r, 0xff); + b.bin(b.i64, B::Mul, byte, b.k(b.i64, 0x0101_0101_0101_0101)) + } + 1 => { + let h = b.imm(b.i64, B::BitAnd, r, 0xffff); + b.bin(b.i64, B::Mul, h, b.k(b.i64, 0x0001_0001_0001_0001)) + } + 2 => { + let w = b.imm(b.i64, B::BitAnd, r, 0xffff_ffff); + b.bin(b.i64, B::BitOr, w, b.imm(b.i64, B::Shl, w, 32)) + } + _ => r, + }; + let at = b.v_at(rd); + b.store(at, val64); + let hi = if q { val64 } else { b.k(b.i64, 0) }; + b.store(at + 8, hi); +} +pub(crate) fn ext(b: &Builder, a: SimdExt) { + let at_d = b.v_at(a.rd); + let at_n = b.v_at(a.rn); + let at_m = b.v_at(a.rm); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + let vm_lo = b.load(b.i64, at_m); + let vm_hi = b.load(b.i64, at_m + 8); + let (lo, hi) = if a.imm == 0 { + (vn_lo, vn_hi) + } else if a.imm == 8 { + (vn_hi, vm_lo) + } else if a.imm < 8 { + let s = a.imm as u64 * 8; + let r = (8 - a.imm as u64) * 8; + let lo = b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vn_lo, s), + b.imm(b.i64, B::Shl, vn_hi, r), + ); + let hi = b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vn_hi, s), + b.imm(b.i64, B::Shl, vm_lo, r), + ); + (lo, hi) + } else { + let shift = a.imm as u64 - 8; + let s = shift * 8; + let r = (8 - shift) * 8; + let lo = b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vn_hi, s), + b.imm(b.i64, B::Shl, vm_lo, r), + ); + let hi = b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vm_lo, s), + b.imm(b.i64, B::Shl, vm_hi, r), + ); + (lo, hi) + }; + b.store(at_d, lo); + b.store(at_d + 8, hi); +} +pub(crate) fn compare_zero(b: &Builder, a: SimdCompareZero) { + let at_n = b.v_at(a.rn); + let at_d = b.v_at(a.rd); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + let ebits = (1u64 << a.size) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let halves = if a.q { 2 } else { 1 }; + for half in 0..2 { + if half >= halves { + b.store(at_d + half * 8, b.k(b.i64, 0)); + continue; + } + let v = if half == 0 { vn_lo } else { vn_hi }; + let mut acc = b.k(b.i64, 0); + let lanes = 8 >> a.size; + for lane in 0..lanes { + let shift = lane as u64 * ebits; + let field = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, v, shift), emask); + let is_zero = b.cmp(C::Eq, field, b.k(b.i64, 0)); + let bits = b.sel(b.i64, is_zero, b.k(b.i64, emask), b.k(b.i64, 0)); + acc = b.bin(b.i64, B::BitOr, acc, b.imm(b.i64, B::Shl, bits, shift)); + } + b.store(at_d + half * 8, acc); + } +} +pub(crate) fn table(b: &Builder, a: SimdTable) { + // Table bytes live in `len + 1` consecutive registers. Each index + // byte selects a table byte; out-of-range indices leave zero for + // `tbl` and keep the destination byte for `tbx`. + let at_m = b.v_at(a.rm); + let vm_lo = b.load(b.i64, at_m); + let vm_hi = b.load(b.i64, at_m + 8); + let regs = a.len as usize + 1; + let mut table = Vec::with_capacity(regs * 2); + for i in 0..regs { + let reg = (a.rn as usize + i) % 32; + let at = b.v_at(reg as u8); + table.push(b.load(b.i64, at)); + table.push(b.load(b.i64, at + 8)); + } + let at_d = b.v_at(a.rd); + let old_lo = if a.extend { + Some(b.load(b.i64, at_d)) + } else { + None + }; + let old_hi = if a.extend { + Some(b.load(b.i64, at_d + 8)) + } else { + None + }; + let halves = if a.q { 2 } else { 1 }; + let limit = (16 * regs) as u64; + for half in 0..2 { + if half >= halves { + b.store(at_d + half * 8, b.k(b.i64, 0)); + continue; + } + let indices = if half == 0 { vm_lo } else { vm_hi }; + let old = if half == 0 { old_lo } else { old_hi }; + let mut acc = b.k(b.i64, 0); + for byte in 0..8u64 { + let shift = byte * 8; + let index = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, indices, shift), 0xff); + let in_range = b.cmp(C::Ult, index, b.k(b.i64, limit)); + let mut value = b.k(b.i64, 0); + let sel = |c: Value, x: Value, y: Value| b.sel(b.i64, c, x, y); + for (slot, entry) in table.iter().enumerate() { + let here = b.cmp( + C::Eq, + b.bin(b.i64, B::Shr, index, b.k(b.i64, 3)), + b.k(b.i64, slot as u64), + ); + let byte_at = b.imm( + b.i64, + B::BitAnd, + b.bin( + b.i64, + B::Shr, + *entry, + b.bin( + b.i64, + B::Shl, + b.imm(b.i64, B::BitAnd, index, 7), + b.k(b.i64, 3), + ), + ), + 0xff, + ); + value = sel(b.bin(b.boolean, B::BitAnd, in_range, here), byte_at, value); + } + if a.extend { + value = sel( + in_range, + value, + b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, old.unwrap(), shift), + 0xff, + ), + ); + } + acc = b.bin(b.i64, B::BitOr, acc, b.imm(b.i64, B::Shl, value, shift)); + } + b.store(at_d + half * 8, acc); + } +} +pub(crate) fn fmov(b: &Builder, a: SimdFmov) { + // Pure bit move; no FP arithmetic involved. + if a.to_fp { + let v = b.reg(b.i64, a.rn, true); + let bits = if a.double { + v + } else { + b.imm(b.i64, B::BitAnd, v, 0xffff_ffff) + }; + let at = b.v_at(a.rd); + b.store(at, bits); + b.store(at + 8, b.k(b.i64, 0)); + } else { + let at = b.v_at(a.rn); + let v = b.load(b.i64, at); + let bits = if a.double { + v + } else { + b.imm(b.i64, B::BitAnd, v, 0xffff_ffff) + }; + b.put(a.rd, bits); + } +} +pub(crate) fn alu(b: &Builder, a: SimdAlu) { + // Bitwise logical operations work on whole halves, lane width free. + if matches!( + a.op, + SimdAluOp::AddV | SimdAluOp::UMinV | SimdAluOp::UMaxV | SimdAluOp::UAddLV + ) { + // Horizontal add, minimum, maximum or widening add: reduce every + // lane of the source and store the result in lane 0, zeroing the + // rest. `UAddLV` keeps twice the lane width. + let ebits = (1u64 << a.size) * 8; + let mask_of = |bits: u64| { + if bits >= 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + } + }; + let emask = mask_of(ebits); + let result_mask = if a.op == SimdAluOp::UAddLV { + mask_of(ebits * 2) + } else { + emask + }; + let at_n = b.v_at(a.rn); + let at_d = b.v_at(a.rd); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + let halves = if a.q { 2 } else { 1 }; + let mut total = if a.op == SimdAluOp::UMinV { + b.k(b.i64, emask) + } else { + b.k(b.i64, 0) + }; + for half in 0..halves { + let v = if half == 0 { vn_lo } else { vn_hi }; + for lane in 0..(8 >> a.size) { + let shift = lane as u64 * ebits; + let field = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, v, shift), emask); + total = match a.op { + SimdAluOp::UMinV => b.sel(b.i64, b.cmp(C::Ult, field, total), field, total), + SimdAluOp::UMaxV => b.sel(b.i64, b.cmp(C::Ugt, field, total), field, total), + _ => b.bin(b.i64, B::Add, total, field), + }; + } + } + b.store( + at_d, + b.bin(b.i64, B::BitAnd, total, b.k(b.i64, result_mask)), + ); + b.store(at_d + 8, b.k(b.i64, 0)); + } else if matches!( + a.op, + SimdAluOp::And + | SimdAluOp::Bic + | SimdAluOp::Orr + | SimdAluOp::Orn + | SimdAluOp::Eor + | SimdAluOp::Bsl + | SimdAluOp::Bit + | SimdAluOp::Bif + ) { + let at_n = b.v_at(a.rn); + let at_m = b.v_at(a.rm); + let at_d = b.v_at(a.rd); + for half in 0..2 { + if half == 1 && !a.q { + b.store(at_d + 8, b.k(b.i64, 0)); + continue; + } + let x = b.load(b.i64, at_n + half * 8); + let y = b.load(b.i64, at_m + half * 8); + let not = |v| b.bin(b.i64, B::BitXor, v, b.k(b.i64, u64::MAX)); + let r = match a.op { + SimdAluOp::And => b.bin(b.i64, B::BitAnd, x, y), + SimdAluOp::Bic => b.bin(b.i64, B::BitAnd, x, not(y)), + SimdAluOp::Orr => b.bin(b.i64, B::BitOr, x, y), + SimdAluOp::Orn => b.bin(b.i64, B::BitOr, x, not(y)), + SimdAluOp::Eor => b.bin(b.i64, B::BitXor, x, y), + // The destination is both a source mask and the + // merge base for BSL/BIT/BIF. + _ => { + let d = b.load(b.i64, at_d + half * 8); + match a.op { + SimdAluOp::Bsl => b.bin( + b.i64, + B::BitOr, + b.bin(b.i64, B::BitAnd, x, d), + b.bin(b.i64, B::BitAnd, y, not(d)), + ), + _ => { + let selected = match a.op { + SimdAluOp::Bit => { + b.bin(b.i64, B::BitAnd, b.bin(b.i64, B::BitXor, d, x), y) + } + _ => b.bin(b.i64, B::BitAnd, b.bin(b.i64, B::BitXor, d, x), not(y)), + }; + b.bin(b.i64, B::BitXor, d, selected) + } + } + } + }; + b.store(at_d + half * 8, r); + } + } else { + let at_n = b.v_at(a.rn); + let at_m = b.v_at(a.rm); + let at_d = b.v_at(a.rd); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + let vm_lo = b.load(b.i64, at_m); + let vm_hi = b.load(b.i64, at_m + 8); + let need_d = matches!(a.op, SimdAluOp::Mla | SimdAluOp::Mls); + // Pairwise forms consume adjacent lanes of the full source pair: + // lane 0/1 of Vn and Vm, then lane 2/3, and so on. + if matches!( + a.op, + SimdAluOp::UMaxP + | SimdAluOp::SMaxP + | SimdAluOp::UMinP + | SimdAluOp::SMinP + | SimdAluOp::AddP + ) { + let ebits = (1u64 << a.size) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let at_n = b.v_at(a.rn); + let at_m = b.v_at(a.rm); + let at_d = b.v_at(a.rd); + let ssel = |c: Value, x: Value, y: Value| b.sel(b.i64, c, x, y); + let sources = [ + b.load(b.i64, at_n), + b.load(b.i64, at_n + 8), + b.load(b.i64, at_m), + b.load(b.i64, at_m + 8), + ]; + // Each source contributes `per_source` lanes (a 64-bit vector for + // the D form, a 128-bit one for the Q form). Lane `2k` pairs with + // lane `2k + 1` and the result lands in result lane + // `which * reduced + k`: the reduction of Vn fills the low half of + // the result and Vm the high half. For the D form the whole result + // is 64 bits, so Vm's lanes land in bits 32..64. + let per_source = ((if a.q { 128 } else { 64 }) / ebits) as usize; + let reduced_lanes = per_source / 2; + let signed = matches!(a.op, SimdAluOp::SMaxP | SimdAluOp::SMinP); + let mut out = [b.k(b.i64, 0); 2]; + for (which, half) in [(0usize, 0usize), (2usize, 1usize)] { + for k in 0..reduced_lanes { + let lanes_per_dw = (64 / ebits) as usize; + let read = |lane: usize| { + let double = lane / lanes_per_dw; + let shift = (lane % lanes_per_dw) as u64 * ebits; + let raw = b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, sources[which + double], shift), + emask, + ); + if signed { + // Ordered as signed numbers: sign-extend the lane. + let up = 64 - ebits; + b.imm(b.i64, B::Sar, b.imm(b.i64, B::Shl, raw, up), up) + } else { + raw + } + }; + let lo = read(2 * k); + let hi = read(2 * k + 1); + let r = match a.op { + SimdAluOp::AddP => b.bin(b.i64, B::Add, lo, hi), + SimdAluOp::UMaxP => ssel(b.cmp(C::Ugt, lo, hi), lo, hi), + SimdAluOp::UMinP => ssel(b.cmp(C::Ult, lo, hi), lo, hi), + SimdAluOp::SMaxP => ssel(b.cmp(C::Sgt, lo, hi), lo, hi), + _ => ssel(b.cmp(C::Slt, lo, hi), lo, hi), + }; + let masked = b.bin(b.i64, B::BitAnd, r, b.k(b.i64, emask)); + // Result lane `index` sits at bit `index * ebits` of the + // 128-bit result. + let index = half * reduced_lanes + k; + let bit = index as u64 * ebits; + let slot = (bit / 64) as usize; + out[slot] = b.bin( + b.i64, + B::BitOr, + out[slot], + b.imm(b.i64, B::Shl, masked, bit % 64), + ); + } + } + b.store(at_d, out[0]); + b.store(at_d + 8, out[1]); + } else { + let vd_lo = if need_d { + Some(b.load(b.i64, at_d)) + } else { + None + }; + let vd_hi = if need_d { + Some(b.load(b.i64, at_d + 8)) + } else { + None + }; + let ebits = (1u64 << a.size) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let smax = u64::MAX >> (65 - ebits); + let smin = !smax; + let halves = if a.q { 2 } else { 1 }; + for half in 0..2 { + if half >= halves { + b.store(at_d + half * 8, b.k(b.i64, 0)); + continue; + } + let vn = if half == 0 { vn_lo } else { vn_hi }; + let vm = if half == 0 { vm_lo } else { vm_hi }; + let vd = if half == 0 { vd_lo } else { vd_hi }; + let mut acc = b.k(b.i64, 0); + let lanes = 8 >> a.size; + for lane in 0..lanes { + let shift = lane as u64 * ebits; + let fld = |v| b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, v, shift), emask); + let ext = |v: Value| { + let up = b.bin(b.i64, B::Shl, v, b.k(b.i64, 64 - ebits)); + b.imm(b.i64, B::Sar, up, 64 - ebits) + }; + let an = fld(vn); + let bn = fld(vm); + let sa = ext(an); + let sb = ext(bn); + let dn = vd.map(fld); + let ssel = |c: Value, x: Value, y: Value| b.sel(b.i64, c, x, y); + let ssat = |v: Value| { + let over = b.cmp(C::Sgt, v, b.k(b.i64, smax)); + let under = b.cmp(C::Slt, v, b.k(b.i64, smin)); + ssel(over, b.k(b.i64, smax), ssel(under, b.k(b.i64, smin), v)) + }; + let r = match a.op { + SimdAluOp::Add => b.bin(b.i64, B::Add, an, bn), + SimdAluOp::Sub => b.bin(b.i64, B::Sub, an, bn), + SimdAluOp::Mul => b.bin(b.i64, B::Mul, an, bn), + SimdAluOp::Mla => { + b.bin(b.i64, B::Add, dn.unwrap(), b.bin(b.i64, B::Mul, an, bn)) + } + SimdAluOp::Mls => { + b.bin(b.i64, B::Sub, dn.unwrap(), b.bin(b.i64, B::Mul, an, bn)) + } + SimdAluOp::CmEq => { + ssel(b.cmp(C::Eq, an, bn), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmGt => { + ssel(b.cmp(C::Sgt, sa, sb), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmGe => { + ssel(b.cmp(C::Sge, sa, sb), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmHi => { + ssel(b.cmp(C::Ugt, an, bn), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmHs => { + ssel(b.cmp(C::Uge, an, bn), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmTst => ssel( + b.cmp(C::Ne, b.bin(b.i64, B::BitAnd, an, bn), b.k(b.i64, 0)), + b.k(b.i64, emask), + b.k(b.i64, 0), + ), + SimdAluOp::UMin => ssel(b.cmp(C::Ult, an, bn), an, bn), + SimdAluOp::UMax => ssel(b.cmp(C::Ugt, an, bn), an, bn), + SimdAluOp::SMin => ssel(b.cmp(C::Slt, sa, sb), an, bn), + SimdAluOp::SMax => ssel(b.cmp(C::Sgt, sa, sb), an, bn), + SimdAluOp::UAbd => { + let d = b.bin(b.i64, B::Sub, an, bn); + ssel( + b.cmp(C::Uge, an, bn), + d, + b.bin(b.i64, B::Sub, b.k(b.i64, 0), d), + ) + } + SimdAluOp::SAbd => { + let d = b.bin(b.i64, B::Sub, sa, sb); + let neg = b.bin(b.i64, B::Sub, b.k(b.i64, 0), d); + ssel(b.cmp(C::Sge, sa, sb), d, neg) + } + SimdAluOp::SQAdd => ssat(b.bin(b.i64, B::Add, sa, sb)), + SimdAluOp::UQAdd => { + let s = b.bin(b.i64, B::Add, an, bn); + let carry = b.cmp(C::Ult, s, an); + let big = b.cmp(C::Ugt, s, b.k(b.i64, emask)); + ssel(b.bin(b.i64, B::BitOr, carry, big), b.k(b.i64, emask), s) + } + SimdAluOp::SQSub => ssat(b.bin(b.i64, B::Sub, sa, sb)), + SimdAluOp::UQSub => { + let borrow = b.cmp(C::Ugt, bn, an); + ssel(borrow, b.k(b.i64, 0), b.bin(b.i64, B::Sub, an, bn)) + } + SimdAluOp::SHAdd + | SimdAluOp::UHAdd + | SimdAluOp::SHSub + | SimdAluOp::UHSub + | SimdAluOp::SRHAdd + | SimdAluOp::URHAdd => { + let signed_op = matches!( + a.op, + SimdAluOp::SHAdd | SimdAluOp::SHSub | SimdAluOp::SRHAdd + ); + // Signed forms operate on sign-extended lanes, which + // keeps the halving shift meaningful for negatives. + let (op_a, op_b) = if signed_op { (sa, sb) } else { (an, bn) }; + let shr = |v: Value| { + if signed_op { + b.imm(b.i64, B::Sar, v, 1) + } else { + b.imm(b.i64, B::Shr, v, 1) + } + }; + let k = b.bin(b.i64, B::Sub, shr(op_a), shr(op_b)); + let k = if matches!( + a.op, + SimdAluOp::SHAdd + | SimdAluOp::UHAdd + | SimdAluOp::SRHAdd + | SimdAluOp::URHAdd + ) { + b.bin(b.i64, B::Add, shr(op_a), shr(op_b)) + } else { + k + }; + let a0 = b.bin(b.i64, B::BitAnd, op_a, b.k(b.i64, 1)); + let b0 = b.bin(b.i64, B::BitAnd, op_b, b.k(b.i64, 1)); + let corr = match a.op { + SimdAluOp::SHAdd | SimdAluOp::UHAdd => { + b.bin(b.i64, B::BitAnd, a0, b0) + } + SimdAluOp::SRHAdd | SimdAluOp::URHAdd => { + b.bin(b.i64, B::BitOr, a0, b0) + } + _ => b.bin( + b.i64, + B::BitAnd, + b.bin(b.i64, B::BitXor, a0, b.k(b.i64, 1)), + b0, + ), + }; + if matches!( + a.op, + SimdAluOp::SHAdd + | SimdAluOp::UHAdd + | SimdAluOp::SRHAdd + | SimdAluOp::URHAdd + ) { + b.bin(b.i64, B::Add, k, corr) + } else { + b.bin(b.i64, B::Sub, k, corr) + } + } + SimdAluOp::SSHl + | SimdAluOp::USHl + | SimdAluOp::SQShl + | SimdAluOp::UQShl + | SimdAluOp::SQRShl + | SimdAluOp::UQRShl => { + let signed_op = matches!( + a.op, + SimdAluOp::SSHl | SimdAluOp::SQShl | SimdAluOp::SQRShl + ); + let rounding = matches!(a.op, SimdAluOp::SQRShl | SimdAluOp::UQRShl); + let saturating = matches!( + a.op, + SimdAluOp::SQShl + | SimdAluOp::UQShl + | SimdAluOp::SQRShl + | SimdAluOp::UQRShl + ); + // Shift amount: the lane's low byte, sign-extended to + // 64 bits (negative means a right shift). + let amt = b.bin( + b.i64, + B::Shl, + b.bin(b.i64, B::BitAnd, bn, b.k(b.i64, 0xff)), + b.k(b.i64, 56), + ); + let amt = b.bin(b.i64, B::Sar, amt, b.k(b.i64, 56)); + let left = b.cmp(C::Sge, amt, b.k(b.i64, 0)); + let n = ssel(left, amt, b.bin(b.i64, B::Sub, b.k(b.i64, 0), amt)); + let shl = |v: Value, k: Value| b.bin(b.i64, B::Shl, v, k); + let sar = |v: Value, k: Value| b.bin(b.i64, B::Sar, v, k); + let shr = |v: Value, k: Value| b.bin(b.i64, B::Shr, v, k); + // Beyond the lane width: a plain right shift keeps the + // sign fill (or zero), a rounding shift collapses to zero, + // and a saturating left shift pins to the lane bound. + let sign_fill = if signed_op { + ssel( + b.cmp(C::Slt, sa, b.k(b.i64, 0)), + b.k(b.i64, emask), + b.k(b.i64, 0), + ) + } else { + b.k(b.i64, 0) + }; + let sat_fill = if signed_op { + ssel( + b.cmp(C::Slt, sa, b.k(b.i64, 0)), + b.k(b.i64, smin), + b.k(b.i64, smax), + ) + } else { + b.k(b.i64, emask) + }; + let far = ssel( + left, + if saturating { sat_fill } else { b.k(b.i64, 0) }, + if rounding { b.k(b.i64, 0) } else { sign_fill }, + ); + // Shift when the amount stays inside the lane width. + // Clamping to `ebits - 1` keeps the host shift defined. + let kc = ssel( + b.cmp(C::Uge, n, b.k(b.i64, ebits)), + b.k(b.i64, ebits - 1), + n, + ); + let rnd = if rounding { + shl(b.k(b.i64, 1), b.bin(b.i64, B::Sub, kc, b.k(b.i64, 1))) + } else { + b.k(b.i64, 0) + }; + let opnd = if signed_op { sa } else { an }; + let t = if rounding { + // The rounding constant must not leak out of the lane. + b.bin( + b.i64, + B::BitAnd, + b.bin(b.i64, B::Add, opnd, rnd), + b.k(b.i64, emask), + ) + } else { + opnd + }; + let right = if rounding { + if signed_op { sar(t, kc) } else { shr(t, kc) } + } else if signed_op { + sar(opnd, kc) + } else { + shr(opnd, kc) + }; + let shifted = shl(opnd, kc); + let moved = ssel(left, shifted, right); + let moved = ssel(b.cmp(C::Uge, n, b.k(b.i64, ebits)), far, moved); + if !saturating { + moved + } else if signed_op { + // Saturation only matters for left shifts; the + // right-shift result already fits the lane. Below + // the 64-bit lane the shifted value is exact, so the + // lane bounds test it directly; a 64-bit lane needs + // the round-trip check because the shift wraps. + let ov = if ebits == 64 { + b.cmp(C::Ne, sar(shifted, kc), opnd) + } else { + b.bin( + b.boolean, + B::BitOr, + b.cmp(C::Sgt, shifted, b.k(b.i64, smax)), + b.cmp(C::Slt, shifted, b.k(b.i64, smin)), + ) + }; + let sat = ssel( + b.cmp(C::Slt, sa, b.k(b.i64, 0)), + b.k(b.i64, smin), + b.k(b.i64, smax), + ); + ssel(left, ssel(ov, sat, moved), moved) + } else { + let ov = if ebits == 64 { + b.cmp(C::Ne, shr(shifted, kc), an) + } else { + b.cmp(C::Ugt, shifted, b.k(b.i64, emask)) + }; + ssel(left, ssel(ov, b.k(b.i64, emask), moved), moved) + } + } + _ => b.k(b.i64, 0), + }; + let masked = b.bin(b.i64, B::BitAnd, r, b.k(b.i64, emask)); + acc = b.bin(b.i64, B::BitOr, acc, b.imm(b.i64, B::Shl, masked, shift)); + } + b.store(at_d + half * 8, acc); + } + } + } +} diff --git a/src/lift/aarch64/src/tests.rs b/src/lift/aarch64/src/tests.rs new file mode 100644 index 0000000..be95711 --- /dev/null +++ b/src/lift/aarch64/src/tests.rs @@ -0,0 +1,949 @@ +//! Tests lift real encodings with a flat test environment and run them on the +//! host through the native JIT. +use crate::{Builder, Environment, LiftError, Lifter, MemoryAccess}; +use alloc::vec::Vec; +use core::mem::offset_of; +use volt_ir::function::{BinOp as B, Function, Value}; +use volt_target::{ + aarch64::decode::*, + native::{self, JittedFunction}, +}; + +/// Guest state of the test environment. +#[repr(C)] +#[derive(Default)] +struct State { + x: [u64; 31], + sp: u64, + flags: u32, + v: [[u64; 2]; 32], +} + +/// Memory is the host's own: a guest address is a host pointer and every +/// load or store is an ordinary IR load or store. Counts the lifter's hooks. +#[derive(Default)] +struct Flat { + begun: usize, + exits: usize, +} +impl Flat { + /// `address + delta` as a host pointer. + fn host(b: &Builder, address: Value, delta: u64) -> Value { + b.imm(b.ptr, B::Add, address, delta) + } + fn load(b: &Builder, address: Value, delta: u64, size: Size, rt: u8, signed: SignExtend) { + let bits = size as u16 * 8; + let raw = b.load_ptr(b.int(bits), Self::host(b, address, delta)); + let mut v = b.cv(b.i64, raw); + if signed != SignExtend::None && bits < 64 { + let n = 64 - bits as u64; + let extended = b.shift(Width::X64, b.imm(b.i64, B::Shl, v, n), Shift::Asr, n); + v = if signed == SignExtend::To32 { + b.imm(b.i64, B::BitAnd, extended, 0xffff_ffff) + } else { + extended + }; + } + b.put(rt, v); + } + fn store(b: &Builder, address: Value, delta: u64, size: Size, rt: u8) { + let t = b.int(size as u16 * 8); + b.store_ptr( + Self::host(b, address, delta), + b.cv(t, b.reg(b.i64, rt, true)), + ); + } + fn vector(b: &Builder, address: Value, bytes: u8, rt: u8, op: MemoryOp, delta: u64) { + let at = b.v_at(rt); + let chunk = b.int(bytes.min(8) as u16 * 8); + let chunks = if bytes == 16 { 2 } else { 1 }; + for i in 0..chunks { + let host = Self::host(b, address, delta + i as u64 * 8); + if op == MemoryOp::Store { + b.store_ptr(host, b.cv(chunk, b.load(b.i64, at + i * 8))); + } else { + b.store(at + i * 8, b.cv(b.i64, b.load_ptr(chunk, host))); + } + } + if op == MemoryOp::Load && bytes <= 8 { + b.store(at + 8, b.k(b.i64, 0)); + } + } +} +impl Environment for Flat { + fn x_offset(&self, r: u8) -> usize { + offset_of!(State, x) + r as usize * 8 + } + fn sp_offset(&self) -> usize { + offset_of!(State, sp) + } + fn flags_offset(&self) -> usize { + offset_of!(State, flags) + } + fn v_offset(&self, r: u8) -> usize { + offset_of!(State, v) + r as usize * 16 + } + fn begin_instruction(&mut self, _: &Builder) { + self.begun += 1; + } + fn exit(&mut self, b: &Builder, next_pc: Value) { + self.exits += 1; + b.set_ret(next_pc) + } + fn memory(&mut self, b: &Builder, _: u64, access: MemoryAccess<'_>) -> Result<(), LiftError> { + let MemoryAccess { + insn, + address, + writeback, + } = access; + match (*insn, address) { + (Instruction::Memory(a), Some(address)) if a.op == MemoryOp::Store => { + Self::store(b, address, 0, a.size, a.rt) + } + (Instruction::Memory(a), Some(address)) => { + Self::load(b, address, 0, a.size, a.rt, a.signed) + } + (Instruction::Literal(a), Some(address)) => { + Self::load(b, address, 0, a.size, a.rt, SignExtend::None) + } + (Instruction::Pair(a), Some(address)) if a.op == MemoryOp::Store => { + Self::store(b, address, 0, a.size, a.rt); + Self::store(b, address, a.size as u64, a.size, a.rt2); + } + (Instruction::Pair(a), Some(address)) => { + Self::load(b, address, 0, a.size, a.rt, a.signed); + Self::load(b, address, a.size as u64, a.size, a.rt2, a.signed); + } + (Instruction::SimdMemory(a), Some(address)) => { + Self::vector(b, address, a.bytes, a.rt, a.op, 0) + } + (Instruction::SimdLiteral(a), Some(address)) => { + Self::vector(b, address, a.bytes, a.rt, MemoryOp::Load, 0) + } + (Instruction::SimdPair(a), Some(address)) => { + Self::vector(b, address, a.bytes, a.rt, a.op, 0); + Self::vector(b, address, a.bytes, a.rt2, a.op, a.bytes as u64); + } + _ => return Err(LiftError::Unsupported), + } + if let Some(w) = writeback { + b.put_sp(w.register, w.value); + } + Ok(()) + } + fn event(&mut self, _: &Builder, _: u64, _: &Instruction) -> Result<(), LiftError> { + Err(LiftError::Unsupported) + } +} + +/// Lift `bytes` as one block at `pc`: instructions until one terminates it +/// (plain memory accesses do not), then an exit at the next pc. +fn lift(pc: u64, bytes: &[u8]) -> Result<(Function, Flat), LiftError> { + let mut lifter = Lifter::new(Flat::default()); + let mut at = pc; + let mut ended = false; + for chunk in bytes.chunks_exact(4) { + let word = u32::from_le_bytes(chunk.try_into().unwrap()); + let instruction = decode(word).unwrap(); + ended = instruction.terminates_with(true); + lifter.lift(at, instruction)?; + at += 4; + if ended { + break; + } + } + if !ended { + lifter.exit(at); + } + Ok(lifter.into_parts()) +} +struct Block { + _image: JittedFunction, + entry: unsafe extern "C" fn(*mut State) -> u64, +} +impl Block { + fn run(&self, state: &mut State) -> u64 { + // SAFETY: the lifted function has exactly this signature and `_image` + // keeps its mapping alive. + unsafe { (self.entry)(state) } + } +} +fn compile(pc: u64, bytes: &[u8]) -> Result { + let (function, _) = lift(pc, bytes)?; + let image = native::compile_owned(function).unwrap(); + // SAFETY: see `Block::run`. + let entry = unsafe { image.entry() }; + Ok(Block { + _image: image, + entry, + }) +} +fn bytes(words: &[u32]) -> Vec { + words.iter().flat_map(|w| w.to_le_bytes()).collect() +} + +#[test] +fn zero_division_and_signed_overflow_do_not_trap_host() { + // UDIV X2,X0,X1; SDIV X3,X0,X1. + let block = compile(0, &bytes(&[0x9ac10802, 0x9ac10c03])).unwrap(); + let mut cpu = State::default(); + cpu.x[0] = 1 << 63; + cpu.x[1] = 0; + block.run(&mut cpu); + assert_eq!(cpu.x[2], 0); + assert_eq!(cpu.x[3], 0); + cpu.x[1] = u64::MAX; + block.run(&mut cpu); + assert_eq!(cpu.x[3], 1 << 63); +} +#[test] +fn signed_and_unsigned_integer_ops_select_distinct_semantics() { + // sdiv/udiv/asrv (w and x), smulh/umulh, sxtw, sbfx, asr #imm, then + // `cmp` + `cset` for each signed/unsigned condition on w and x. + let words = [ + 0x1ac10c02, 0x1ac10803, 0x1ac12804, 0x9ac12805, 0x9b417c06, 0x9bc17c07, 0x93407c08, + 0x93442c09, 0x13037c0a, 0x6b01001f, 0x1a9fa7eb, 0x1a9fb7ec, 0x1a9f27ed, 0x1a9f97ee, + 0x1a9fd7ef, 0x1a9fc7f0, 0x1a9f87f1, 0xeb01001f, 0x9a9fa7f2, 0x9a9f27f3, 0x9a9fd7f4, + ]; + let block = compile(0, &bytes(&words)).unwrap(); + let pairs: [(u64, u64); 9] = [ + (0xffff_ffff_8000_0005, 3), + (0x8000_0000, 0xffff_ffff), + (0x7fff_ffff_ffff_ffff, 0x8000_0000_0000_0000), + (0x8000_0000_0000_0000, u64::MAX), + (3, 0), + (7, 7), + (0xffff_ffff_0000_0007, 0x0000_0001_0000_0009), + (0x8000_0001, 2), + (2, 0x8000_0001), + ]; + for (a, b) in pairs { + let mut cpu = State::default(); + cpu.x[0] = a; + cpu.x[1] = b; + block.run(&mut cpu); + let (w, wb) = (a as u32, b as u32); + let (sw, swb) = (w as i32, wb as i32); + let sdiv = if wb == 0 { + 0 + } else { + sw.wrapping_div(swb) as u32 + }; + let udiv = if wb == 0 { 0 } else { w / wb }; + assert_eq!(cpu.x[2], sdiv as u64, "sdiv w {a:#x} {b:#x}"); + assert_eq!(cpu.x[3], udiv as u64, "udiv w {a:#x} {b:#x}"); + assert_eq!(cpu.x[4], (sw >> (wb & 31)) as u32 as u64, "asr w"); + assert_eq!(cpu.x[5], ((a as i64) >> (b & 63)) as u64, "asr x"); + assert_eq!( + cpu.x[6], + ((a as i64 as i128 * b as i64 as i128) >> 64) as u64, + "smulh" + ); + assert_eq!(cpu.x[7], ((a as u128 * b as u128) >> 64) as u64, "umulh"); + assert_eq!(cpu.x[8], w as i32 as i64 as u64, "sxtw"); + assert_eq!(cpu.x[9], ((a >> 4) as u8) as i8 as i64 as u64, "sbfx"); + assert_eq!(cpu.x[10], (sw >> 3) as u32 as u64, "asr w imm"); + // Registers 11-17 come from the 32-bit compare, 18-20 from the 64-bit one. + let w_flags = [ + sw < swb, + sw >= swb, + w < wb, + w > wb, + sw > swb, + sw <= swb, + w <= wb, + ]; + for (i, want) in w_flags.into_iter().enumerate() { + assert_eq!(cpu.x[11 + i], want as u64, "w cond {i} {a:#x} {b:#x}"); + } + let (sa, sb) = (a as i64, b as i64); + assert_eq!(cpu.x[18], (sa < sb) as u64, "x lt"); + assert_eq!(cpu.x[19], (a < b) as u64, "x lo"); + assert_eq!(cpu.x[20], (sa > sb) as u64, "x gt"); + } +} +#[test] +fn signed_and_unsigned_vector_lanes_select_distinct_semantics() { + // smax.16b, sabd.8h, shadd.4s, sshl.4s, sqadd.16b, cmgt.2d, cmhi.2d. + let words = [ + 0x4e216403, 0x4e617404, 0x4ea10405, 0x4ea14406, 0x4e210c07, 0x4ee13409, 0x6ee1340a, + ]; + let block = compile(0, &bytes(&words)).unwrap(); + let a: [u8; 16] = [ + 0x80, 0x7f, 0x01, 0xff, 0x10, 0x90, 0x00, 0x7e, 0x85, 0x05, 0xfe, 0x40, 0x7f, 0x80, 0xc0, + 0x20, + ]; + let b: [u8; 16] = [ + 0x01, 0x80, 0xff, 0x01, 0x90, 0x10, 0x00, 0x7f, 0x05, 0x85, 0x02, 0xc0, 0x7f, 0x7f, 0x40, + 0xe0, + ]; + let pack = |x: [u8; 16]| { + let v = u128::from_le_bytes(x); + [v as u64, (v >> 64) as u64] + }; + let mut cpu = State::default(); + cpu.v[0] = pack(a); + cpu.v[1] = pack(b); + block.run(&mut cpu); + let out = |r: usize| (u128::from(cpu.v[r][1]) << 64 | u128::from(cpu.v[r][0])).to_le_bytes(); + let lanes16 = |x: [u8; 16]| -> Vec { + x.chunks(2) + .map(|c| i16::from_le_bytes([c[0], c[1]])) + .collect() + }; + let lanes32 = |x: [u8; 16]| -> Vec { + x.chunks(4) + .map(|c| i32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + }; + // smax.16b + let want: Vec = (0..16) + .map(|i| (a[i] as i8).max(b[i] as i8) as u8) + .collect(); + assert_eq!(out(3).to_vec(), want, "smax"); + // sabd.8h + let want: Vec = lanes16(a) + .iter() + .zip(lanes16(b)) + .flat_map(|(x, y)| ((*x as i32 - y as i32).unsigned_abs() as u16).to_le_bytes()) + .collect(); + assert_eq!(out(4).to_vec(), want, "sabd"); + // shadd.4s + let want: Vec = lanes32(a) + .iter() + .zip(lanes32(b)) + .flat_map(|(x, y)| (((*x as i64 + y as i64) >> 1) as i32).to_le_bytes()) + .collect(); + assert_eq!(out(5).to_vec(), want, "shadd"); + // sshl.4s: the shift amount is the signed low byte of each lane of b. + let want: Vec = lanes32(a) + .iter() + .zip(lanes32(b)) + .flat_map(|(x, y)| { + let n = y as i8 as i32; + let r = if n >= 32 { + 0 + } else if n >= 0 { + x << n + } else if n > -32 { + x >> -n + } else if *x < 0 { + -1 + } else { + 0 + }; + r.to_le_bytes() + }) + .collect(); + assert_eq!(out(6).to_vec(), want, "sshl"); + // sqadd.16b + let want: Vec = (0..16) + .map(|i| (a[i] as i8).saturating_add(b[i] as i8) as u8) + .collect(); + assert_eq!(out(7).to_vec(), want, "sqadd"); + // cmgt.2d / cmhi.2d + for (r, signed) in [(9, true), (10, false)] { + for lane in 0..2 { + let (x, y) = (cpu.v[0][lane], cpu.v[1][lane]); + let gt = if signed { + (x as i64) > (y as i64) + } else { + x > y + }; + assert_eq!( + cpu.v[r][lane], + if gt { u64::MAX } else { 0 }, + "cmp {r} {lane}" + ); + } + } +} +#[test] +fn simd_compare_zero_masks_each_lane() { + // cmeq v1.16b, v0.16b, #0. + let block = compile(0x1000, &bytes(&[0x4e209801])).unwrap(); + let mut cpu = State::default(); + cpu.v[0][0] = 0x00ff_0000_0000_00ff; + cpu.v[0][1] = 0xffff_ffff_ffff_ffff; + block.run(&mut cpu); + assert_eq!(cpu.v[1][0], 0xff00_ffff_ffff_ff00); + assert_eq!(cpu.v[1][1], 0); + // Half-width form operates on v0.8b and clears the top half. + let block = compile(0x1000, &bytes(&[0x0e209801])).unwrap(); + cpu.v[0][0] = 0x00ff_0000_0000_00ff; + cpu.v[0][1] = 0x1234_5678_9abc_def0; + cpu.v[1] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; + block.run(&mut cpu); + assert_eq!(cpu.v[1][0], 0xff00_ffff_ffff_ff00); + assert_eq!(cpu.v[1][1], 0); +} +#[test] +fn simd_fmov_moves_bits_between_gpr_and_fpr() { + // fmov x2, d2 then fmov d3, x2 round-trips the low doubleword. + let block = compile(0x1000, &bytes(&[0x9e660042, 0x9e670043])).unwrap(); + let mut cpu = State::default(); + cpu.v[2] = [0x1122_3344_5566_7788, 0xdead_beef_dead_beef]; + block.run(&mut cpu); + assert_eq!(cpu.x[2], 0x1122_3344_5566_7788); + assert_eq!(cpu.v[3], [0x1122_3344_5566_7788, 0]); + // 32-bit form truncates to the low word. + let block = compile(0x1000, &bytes(&[0x1e260022])).unwrap(); + cpu.v[1] = [0xaaaa_bbbb_cccc_dddd, 0]; + block.run(&mut cpu); + assert_eq!(cpu.x[2] as u32, 0xcccc_dddd); +} +#[test] +fn simd_pairwise_reductions_match_lane_layout() { + let setup = || { + let mut cpu = State::default(); + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + cpu.v[2] = [0x1112_1314_1516_1718, 0x191a_1b1c_1d1e_1f20]; + cpu + }; + for (word, want) in [ + ( + 0x6e22a420u32, + [0x0a0c_0e10_0204_0608u64, 0x1a1c_1e20_1214_1618], + ), + (0x6e22ac20, [0x090b_0d0f_0103_0507, 0x191b_1d1f_1113_1517]), + (0x4e22bc20, [0x1317_1b1f_0307_0b0f, 0x3337_3b3f_2327_2b2f]), + (0x4ea2bc20, [0x1618_1a1c_0608_0a0c, 0x3638_3a3c_2628_2a2c]), + (0x4ee2bc20, [0x0a0c_0e10_1214_1618, 0x2a2c_2e30_3234_3638]), + ] { + let mut cpu = setup(); + let block = compile(0x1000, &bytes(&[word])).unwrap(); + block.run(&mut cpu); + assert_eq!(cpu.v[0], want, "{word:08x}"); + } +} +#[test] +fn simd_horizontal_add_sums_lanes() { + for (word, lanes, want) in [ + (0x4e31b820u32, 16usize, 0x88u64), + (0x4e71b820, 8, 0x4048), + (0x4eb1b820, 4, 0x1c20_2428), + ] { + let block = compile(0x1000, &bytes(&[word])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + block.run(&mut cpu); + let _ = lanes; + assert_eq!(cpu.v[0], [want, 0], "{word:08x}"); + } + // The 64-bit form reads only the low half of the source. + let block = compile(0x1000, &bytes(&[0x0e31b820])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + block.run(&mut cpu); + assert_eq!(cpu.v[0], [0x24, 0]); +} +#[test] +fn simd_dup_from_element_replicates_a_lane() { + for (word, want) in [ + // dup v0.4h, v1.h[0] -> element 0x0708 in every halfword. + (0x0e020420u32, [0x0708_0708_0708_0708u64, 0]), + // dup v0.8h, v1.h[7] -> element 0x090a, both halves. + (0x4e1e0420, [0x090a_090a_090a_090a, 0x090a_090a_090a_090a]), + // dup v0.4s, v1.s[1] -> element 0x0102_0304. + (0x4e0c0420, [0x0102_0304_0102_0304, 0x0102_0304_0102_0304]), + // dup v0.2d, v1.d[1] -> element 0x090a_0b0c_0d0e_0f10. + (0x4e180420, [0x090a_0b0c_0d0e_0f10, 0x090a_0b0c_0d0e_0f10]), + // dup v0.16b, v1.b[3] -> element 0x05. + (0x4e070420, [0x0505_0505_0505_0505, 0x0505_0505_0505_0505]), + ] { + let mut cpu = State::default(); + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + let block = compile(0x1000, &bytes(&[word])).unwrap(); + block.run(&mut cpu); + assert_eq!(cpu.v[0], want, "{word:08x}"); + } +} +#[test] +fn simd_table_lookup_selects_bytes() { + // tbl v0.16b, {v1.16b, v2.16b}, v3.16b with indices 0,2,4,... + let block = compile(0x1000, &bytes(&[0x4e032020])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; + cpu.v[2] = [0x1716_1514_1312_1110, 0x1f1e_1d1c_1b1a_1918]; + cpu.v[3] = [0x0604_0200_0e0c_0a08, 0x1614_1210_1e1c_1a18]; + block.run(&mut cpu); + assert_eq!(cpu.v[0], [0x0604_0200_0e0c_0a08, 0x1614_1210_1e1c_1a18]); + // Out-of-range indices yield zero for tbl. + let block = compile(0x1000, &bytes(&[0x4e020020])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; + cpu.v[2] = [0xff01_ff00_ff00_ff00, 0xffff_ffff_ffff_ffff]; + block.run(&mut cpu); + assert_eq!(cpu.v[0], [0x0001_0000_0000_0000, 0]); + // tbx keeps the destination byte where the index is out of range. + let block = compile(0x1000, &bytes(&[0x4e021020])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; + cpu.v[2] = [0xff01_ff00_ff00_ff00, 0xffff_ffff_ffff_ffff]; + cpu.v[0] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; + block.run(&mut cpu); + assert_eq!(cpu.v[0], [0xde01_be00_de00_be00, 0xdead_beef_dead_beef]); +} +#[test] +fn simd_alu_lane_semantics() { + // Ground truth computed with the host's own byte arithmetic. + struct Case { + word: u32, + a: [u64; 2], + b: [u64; 2], + want: [u64; 2], + } + let cases = [ + // add v0.16b, v1.16b, v2.16b wraps per byte + Case { + word: 0x4e228420, + a: [0x0102_03ff_0405_0607, 0x0f10_1120_2122_2324], + b: [0x0101_0101_0101_0101, 0x0102_0304_0506_0708], + want: [0x0203_0400_0506_0708, 0x1012_1424_2628_2a2c], + }, + // sub v0.16b, v1.16b, v2.16b + Case { + word: 0x6e228420, + a: [0x0001_0203_0405_0607, 0], + b: [0x0101_0101_0101_0101, 0], + want: [0xff00_0102_0304_0506, 0], + }, + // cmhs v0.16b, v1.16b, v2.16b (unsigned >=) + Case { + word: 0x6e223c20, + a: [0x00ff_0080_7f7f_8000, 0], + b: [0x0001_0180_7f80_8000, 0], + want: [0xffff_00ff_ff00_ffff, 0xffff_ffff_ffff_ffff], + }, + // umin v0.8h, v1.8h, v2.8h + Case { + word: 0x6e626c20, + a: [0x0003_0002_0001_0000, 0], + b: [0x0002_0003_0000_0001, 0], + want: [0x0002_0002_0000_0000, 0], + }, + // uqadd v0.8h, v1.8h, v2.8h saturates + Case { + word: 0x6e620c20, + a: [0xffff_8000_0001_0000, 0], + b: [0x0001_9000_0002_0001, 0], + want: [0xffff_ffff_0003_0001, 0], + }, + // shadd v0.16b, v1.16b, v2.16b rounds down (arithmetic) + Case { + word: 0x4e220420, + a: [0x0303_fefe_0505_00ff, 0], + b: [0x0303_fefe_0505_00ff, 0], + want: [0x0303_fefe_0505_00ff, 0], + }, + // sshl v0.8h by signed amounts in the low byte of each lane + Case { + word: 0x4e624420, + a: [0x0004_0003_0002_0001, 0], + b: [0x0000_0002_00ff_0001, 0], + want: [0x0004_000c_0001_0002, 0], + }, + // mla/mls fold in the destination *lane*, not the whole half + Case { + word: 0x4e229420, + a: [0x0202_0202_0202_0202, 0], + b: [0x0505_0505_0505_0505, 0], + want: [0x0a0a_0a0a_0a0a_0a0a, 0], + }, + Case { + word: 0x6e229420, + a: [0x0202_0202_0202_0202, 0], + b: [0x0505_0505_0505_0505, 0], + want: [0xf6f6_f6f6_f6f6_f6f6, 0], + }, + // bsl: destination starts zero, so the result is v2 unchanged + Case { + word: 0x6e621c20, + a: [0xaaaa_aaaa_aaaa_aaaa, 0], + b: [0x5555_5555_5555_5555, 0], + want: [0x5555_5555_5555_5555, 0], + }, + // bit with a zero destination inserts the masked v1 bits + Case { + word: 0x6ea21c20, + a: [0x0f0f_0f0f_0f0f_0f0f, 0], + b: [0xaaaa_aaaa_aaaa_aaaa, 0], + want: [0x0a0a_0a0a_0a0a_0a0a, 0], + }, + // bif with a zero destination inserts the inverse-masked bits + Case { + word: 0x6ee21c20, + a: [0x0f0f_0f0f_0f0f_0f0f, 0], + b: [0xaaaa_aaaa_aaaa_aaaa, 0], + want: [0x0505_0505_0505_0505, 0], + }, + // shadd of two -1 bytes stays -1 (arithmetic halving) + Case { + word: 0x4e220420, + a: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], + b: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], + want: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], + }, + // shsub of 0 - 1 halves to -1 + Case { + word: 0x4e222420, + a: [0, 0], + b: [0x0101_0101_0101_0101, 0], + want: [0xffff_ffff_ffff_ffff, 0], + }, + // srhadd of 1 and 2 rounds to 2 + Case { + word: 0x4e221420, + a: [0x0101_0101_0101_0101, 0], + b: [0x0202_0202_0202_0202, 0], + want: [0x0202_0202_0202_0202, 0], + }, + // sshl v0.8h: lane 0 shifts left by 1, lane 1 right by 1 + Case { + word: 0x4e624420, + a: [0x0008_0004_0002_0001, 0], + b: [0x0000_00ff_0001_0001, 0], + want: [0x0008_0002_0004_0002, 0], + }, + // uqshl v0.8h saturates an overflow to the lane maximum + Case { + word: 0x6e624c20, + a: [0x0000_0000_0000_8000, 0], + b: [0x0000_0000_0000_0001, 0], + want: [0x0000_0000_0000_ffff, 0], + }, + // sqshl v0.8h saturates a positive overflow to the lane maximum + Case { + word: 0x4e624c20, + a: [0x0000_0000_0000_4000, 0], + b: [0x0000_0000_0000_0001, 0], + want: [0x0000_0000_0000_7fff, 0], + }, + // sqrshl v0.8h rounds before shifting right + Case { + word: 0x4e625c20, + a: [0x0000_0000_0000_0003, 0], + b: [0x0000_0000_0000_00ff, 0], + want: [0x0000_0000_0000_0002, 0], + }, + ]; + for case in cases { + let block = compile(0x1000, &bytes(&[case.word])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = case.a; + cpu.v[2] = case.b; + block.run(&mut cpu); + assert_eq!(cpu.v[0], case.want, "{:08x}", case.word); + } +} + +fn sub_flags(a: u64, b: u64) -> u32 { + let r = a.wrapping_sub(b); + let (n, z, c) = (r >> 63, r == 0, a >= b); + let v = ((a ^ b) & (a ^ r)) >> 63; + (n as u32) << 31 | (z as u32) << 30 | (c as u32) << 29 | (v as u32) << 28 +} +fn add_flags32(a: u32, b: u32) -> u32 { + let (r, c) = a.overflowing_add(b); + let v = (!(a ^ b) & (a ^ r)) >> 31; + (r >> 31) << 31 | ((r == 0) as u32) << 30 | (c as u32) << 29 | v << 28 +} +#[test] +fn subs_and_adds_set_nzcv() { + // subs x2,x0,x1; adds w3,w0,w1. + let block = compile(0, &bytes(&[0xeb010002, 0x2b010003])).unwrap(); + for (a, b) in [ + (5u64, 5u64), + (3, 5), + (5, 3), + (1 << 63, 1), + (0xffff_ffff, 1), + (0x7fff_ffff, 1), + (u64::MAX, u64::MAX), + ] { + let mut cpu = State::default(); + cpu.x[0] = a; + cpu.x[1] = b; + block.run(&mut cpu); + assert_eq!(cpu.x[2], a.wrapping_sub(b)); + assert_eq!(cpu.x[3], (a as u32).wrapping_add(b as u32) as u64); + assert_eq!( + cpu.flags, + add_flags32(a as u32, b as u32), + "adds {a:#x} {b:#x}" + ); + // The flags of the first instruction were overwritten by the second; + // check them on their own. + let sub = compile(0, &bytes(&[0xeb010002])).unwrap(); + sub.run(&mut cpu); + assert_eq!(cpu.flags, sub_flags(a, b), "subs {a:#x} {b:#x}"); + } +} +#[test] +fn conditional_select_and_compare_read_the_flags() { + // subs x2,x0,x1; csel x4,x0,x1,lt. + let block = compile(0, &bytes(&[0xeb010002, 0x9a81b004])).unwrap(); + for (a, b) in [(3u64, 5u64), (5, 3), (5, 5), (1 << 63, 1)] { + let mut cpu = State::default(); + cpu.x[0] = a; + cpu.x[1] = b; + block.run(&mut cpu); + let want = if (a as i64) < (b as i64) { a } else { b }; + assert_eq!(cpu.x[4], want, "csel lt {a:#x} {b:#x}"); + } + // ccmp x0,x1,#4,eq: compares when EQ holds, otherwise loads the immediate. + let block = compile(0, &bytes(&[0xfa410004])).unwrap(); + let mut cpu = State::default(); + cpu.x[0] = 7; + cpu.x[1] = 9; + cpu.flags = 0x4000_0000; + block.run(&mut cpu); + assert_eq!(cpu.flags, sub_flags(7, 9)); + cpu.flags = 0; + block.run(&mut cpu); + assert_eq!(cpu.flags, 0x4000_0000); +} +#[test] +fn zero_register_destinations_do_not_overwrite_sp() { + // ORR XZR,X0,X0; ADR XZR,. ; ADDS XZR,X0,X0. + let block = compile(0x1000, &bytes(&[0xaa00001f, 0x1000001f, 0xab00001f])).unwrap(); + let mut cpu = State::default(); + cpu.sp = 0x9000; + cpu.x[0] = 0x1234; + block.run(&mut cpu); + assert_eq!(cpu.sp, 0x9000); + // CBZ XZR,+8 must take the branch independently of SP. + let branch = compile(0x2000, &bytes(&[0xb400005f])).unwrap(); + assert_eq!(branch.run(&mut cpu), 0x2008); +} +#[test] +fn branches_return_the_next_pc() { + let mut cpu = State::default(); + cpu.x[0] = 1; + cpu.x[5] = 0xdead_0000; + // b +8; bl +16; br x5; cbnz x0,+8 (taken) and with x0 == 0 (not taken). + assert_eq!( + compile(0x1000, &bytes(&[0x14000002])) + .unwrap() + .run(&mut cpu), + 0x1008 + ); + assert_eq!( + compile(0x1000, &bytes(&[0x94000004])) + .unwrap() + .run(&mut cpu), + 0x1010 + ); + assert_eq!(cpu.x[30], 0x1004); + assert_eq!( + compile(0x1000, &bytes(&[0xd61f00a0])) + .unwrap() + .run(&mut cpu), + 0xdead_0000 + ); + let cbnz = compile(0x1000, &bytes(&[0xb5000040])).unwrap(); + assert_eq!(cbnz.run(&mut cpu), 0x1008); + cpu.x[0] = 0; + assert_eq!(cbnz.run(&mut cpu), 0x1004); +} +#[test] +fn the_environment_sees_every_instruction_and_every_exit() { + // add x0,x0,#1 twice, then fall off the end: two instructions, one exit. + let (_, env) = lift(0x1000, &bytes(&[0x91000400, 0x91000400])).unwrap(); + assert_eq!((env.begun, env.exits), (2, 1)); + // A conditional branch exits once, for both outcomes. + let (_, env) = lift(0x1000, &bytes(&[0xb5000040, 0x91000400])).unwrap(); + assert_eq!((env.begun, env.exits), (1, 1)); +} +#[test] +fn exclusives_and_system_instructions_are_the_environments_business() { + // ldxr x1,[x0]; mrs x1,nzcv. + for word in [0xc85f7c01u32, 0xd53b4201] { + assert_eq!( + lift(0, &bytes(&[word])).err(), + Some(LiftError::Unsupported), + "{word:08x}" + ); + } +} +#[test] +fn plain_loads_and_stores_reach_the_environment_with_their_address() { + let mut buf = [0u64; 8]; + let base = buf.as_mut_ptr() as u64; + let mut cpu = State::default(); + cpu.x[0] = base; + cpu.x[1] = 0x1122_3344_5566_7788; + // str x1,[x0,#16]; ldr x3,[x0,#16]; ldr x4,[x0,x2,lsl #3] with x2 = 2. + cpu.x[2] = 2; + compile(0, &bytes(&[0xf9000801, 0xf9400803, 0xf8627804])) + .unwrap() + .run(&mut cpu); + assert_eq!(buf[2], 0x1122_3344_5566_7788); + assert_eq!(cpu.x[3], 0x1122_3344_5566_7788); + assert_eq!(cpu.x[4], 0x1122_3344_5566_7788); +} +#[test] +fn narrow_loads_extend_as_the_instruction_says() { + let mut buf = [0u64; 1]; + buf[0] = 0xffff_ffff_8765_f0f1; + let mut cpu = State::default(); + for (word, want) in [ + (0x39400001u32, 0xf1u64), // ldrb w1 + (0x39800001, 0xffff_ffff_ffff_fff1), // ldrsb x1 + (0xb9800001, 0xffff_ffff_8765_f0f1), // ldrsw x1 + ] { + cpu.x[0] = buf.as_mut_ptr() as u64; + compile(0, &bytes(&[word])).unwrap().run(&mut cpu); + assert_eq!(cpu.x[1], want, "{word:08x}"); + } + // strh w1,[x0,#2] stores only the low halfword. + cpu.x[0] = buf.as_mut_ptr() as u64; + cpu.x[1] = 0xaaaa_bbbb_cccc_dddd; + compile(0, &bytes(&[0x79000401])).unwrap().run(&mut cpu); + assert_eq!(buf[0], 0xffff_ffff_dddd_f0f1); +} +#[test] +fn pair_accesses_apply_the_writeback_after_the_access() { + let mut buf = [0x11u64, 0x22, 0x33, 0x44]; + let mut cpu = State::default(); + let base = buf.as_mut_ptr() as u64; + // ldp x1,x2,[x0],#16: loads from the old base, then advances it. + cpu.x[0] = base; + compile(0, &bytes(&[0xa8c10801])).unwrap().run(&mut cpu); + assert_eq!((cpu.x[1], cpu.x[2], cpu.x[0]), (0x11, 0x22, base + 16)); + // ldp x1,x2,[x0,#16]!: the access uses the updated base. + cpu.x[0] = base - 16; + compile(0, &bytes(&[0xa9c10801])).unwrap().run(&mut cpu); + assert_eq!((cpu.x[1], cpu.x[2], cpu.x[0]), (0x11, 0x22, base)); + // stp x1,x2,[x0] + cpu.x[0] = base + 16; + cpu.x[1] = 0xaa; + cpu.x[2] = 0xbb; + compile(0, &bytes(&[0xa9000801])).unwrap().run(&mut cpu); + assert_eq!((buf[2], buf[3]), (0xaa, 0xbb)); +} +#[test] +fn literal_loads_use_the_pc_relative_address() { + let buf = [0x0123_4567_89ab_cdefu64, 0xfedc_ba98_7654_3210]; + let mut cpu = State::default(); + // ldr x1,.+8 at a pc that makes the target buf[1]: pc = &buf[0]. + let pc = buf.as_ptr() as u64; + compile(pc, &bytes(&[0x58000041])).unwrap().run(&mut cpu); + assert_eq!(cpu.x[1], 0xfedc_ba98_7654_3210); +} +#[test] +fn vector_loads_and_stores_move_whole_registers() { + let mut buf = [0x1111_2222_3333_4444u64, 0x5555_6666_7777_8888, 0, 0, 0, 0]; + let mut cpu = State::default(); + cpu.x[0] = buf.as_mut_ptr() as u64; + cpu.v[1] = [u64::MAX, u64::MAX]; + // ldr q1,[x0]; str q1,[x0,#16]; ldr d2,[x0,#8] zeroes the top half. + cpu.v[2] = [u64::MAX, u64::MAX]; + compile(0, &bytes(&[0x3dc00001, 0x3d800401, 0xfd400402])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.v[1], [0x1111_2222_3333_4444, 0x5555_6666_7777_8888]); + assert_eq!( + (buf[2], buf[3]), + (0x1111_2222_3333_4444, 0x5555_6666_7777_8888) + ); + assert_eq!(cpu.v[2], [0x5555_6666_7777_8888, 0]); + // ldp q1,q2,[x0] loads two registers: the original pair, and the copy stored above. + cpu.v[1] = [0; 2]; + cpu.v[2] = [0; 2]; + compile(0, &bytes(&[0xad400801])).unwrap().run(&mut cpu); + assert_eq!(cpu.v[1], [0x1111_2222_3333_4444, 0x5555_6666_7777_8888]); + assert_eq!(cpu.v[2], [0x1111_2222_3333_4444, 0x5555_6666_7777_8888]); +} +#[test] +fn simd_pairwise_ops_match_a_reference_model() { + // ADDP/SMAXP/SMINP/UMAXP/UMINP, every element size and both vector widths, + // against random operands. Lane values are compared as the instruction + // defines them: signed forms order the lanes as signed numbers. + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + // (base word, name): Q=0, U=0, size=0, Rm=0, Rn=0, Rd=0. + #[derive(Clone, Copy, PartialEq)] + enum Op { + Add, + SMax, + SMin, + UMax, + UMin, + } + let ops = [Op::Add, Op::SMax, Op::SMin, Op::UMax, Op::UMin]; + for op in ops { + for size in 0..4u32 { + if size == 3 && op != Op::Add { + continue; + } + for q in [false, true] { + if size == 3 && !q { + continue; // ADDP .1D is reserved + } + let (opcode, u) = match op { + Op::Add => (0b10111, 0), + Op::SMax => (0b10100, 0), + Op::SMin => (0b10101, 0), + Op::UMax => (0b10100, 1), + Op::UMin => (0b10101, 1), + }; + let word = 0x0e20_0400u32 + | (q as u32) << 30 + | u << 29 + | size << 22 + | 2 << 16 + | opcode << 11 + | 1 << 5; + let block = compile(0, &bytes(&[word])).unwrap(); + let esize = 8usize << size; + let lanes = (if q { 128 } else { 64 }) / esize; + let mask = if esize == 64 { + u64::MAX + } else { + (1u64 << esize) - 1 + }; + for _ in 0..200 { + let mut cpu = State::default(); + cpu.v[1] = [next(), next()]; + cpu.v[2] = [next(), next()]; + cpu.v[0] = [next(), next()]; // must be fully overwritten + let lane = |v: [u64; 2], i: usize| -> u64 { + let bit = i * esize; + (v[bit / 64] >> (bit % 64)) & mask + }; + let sext = |x: u64| -> i64 { ((x << (64 - esize)) as i64) >> (64 - esize) }; + // Vm:Vn concatenated, `lanes` lanes each. + let all: Vec = (0..lanes) + .map(|i| lane(cpu.v[1], i)) + .chain((0..lanes).map(|i| lane(cpu.v[2], i))) + .collect(); + let mut want = [0u64; 2]; + for i in 0..lanes { + let (a, b) = (all[2 * i], all[2 * i + 1]); + let r = match op { + Op::Add => a.wrapping_add(b) & mask, + Op::SMax => sext(a).max(sext(b)) as u64 & mask, + Op::SMin => sext(a).min(sext(b)) as u64 & mask, + Op::UMax => a.max(b), + Op::UMin => a.min(b), + }; + let bit = i * esize; + want[bit / 64] |= r << (bit % 64); + } + let input = [cpu.v[1], cpu.v[2]]; + block.run(&mut cpu); + assert_eq!( + cpu.v[0], want, + "{word:08x} size {size} q {q} inputs {input:x?}" + ); + } + } + } + } +} From bf600b778c8c39bb240a2d9b690f83a66b3ea94e Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Fri, 9 Oct 2026 17:09:27 -0700 Subject: [PATCH 12/13] feat: add volt-diasm --- Cargo.lock | 15 + Cargo.toml | 3 + README.md | 19 + src/disasm/Cargo.toml | 15 + src/disasm/examples/coverage.rs | 29 + src/disasm/examples/dump.rs | 29 + src/disasm/examples/fmtdiff.rs | 158 ++ src/disasm/src/analysis.rs | 178 ++ src/disasm/src/image.rs | 134 ++ src/disasm/src/lib.rs | 62 + src/emulator/Cargo.toml | 1 + src/emulator/examples/jit_bench.rs | 8 +- src/emulator/src/aarch64/ahead.rs | 58 +- src/emulator/src/aarch64/cache.rs | 8 +- src/emulator/src/aarch64/compile.rs | 19 +- src/emulator/src/aarch64/cpu.rs | 14 +- src/emulator/src/aarch64/environment.rs | 20 +- src/emulator/src/aarch64/fp.rs | 357 +--- src/emulator/src/aarch64/gxf.rs | 27 +- src/emulator/src/aarch64/host.rs | 138 +- src/emulator/src/aarch64/identification.rs | 2 +- src/emulator/src/aarch64/pauth.rs | 384 +--- src/emulator/src/aarch64/simd.rs | 450 +---- src/emulator/src/aarch64/simd_struct.rs | 2 +- src/emulator/src/system/m1.rs | 2 +- src/emulator/src/system/machine.rs | 4 +- src/emulator/src/system/mod.rs | 6 +- src/emulator/src/user/mod.rs | 26 +- src/isa/aarch64/Cargo.toml | 11 + .../src/aarch64 => isa/aarch64/src}/decode.rs | 1488 ++++++++++++--- src/isa/aarch64/src/flow.rs | 137 ++ src/isa/aarch64/src/format/fp.rs | 357 ++++ src/isa/aarch64/src/format/integer.rs | 877 +++++++++ src/isa/aarch64/src/format/memory.rs | 870 +++++++++ src/isa/aarch64/src/format/mod.rs | 211 +++ src/isa/aarch64/src/format/reg.rs | 76 + src/isa/aarch64/src/format/system.rs | 1259 +++++++++++++ src/isa/aarch64/src/format/vector.rs | 1033 +++++++++++ src/isa/aarch64/src/fp.rs | 358 ++++ src/isa/aarch64/src/gxf.rs | 41 + src/isa/aarch64/src/lib.rs | 27 + src/isa/aarch64/src/pauth.rs | 354 ++++ src/isa/aarch64/src/simd.rs | 441 +++++ .../aarch64/src}/simd_struct.rs | 0 src/lift/aarch64/Cargo.toml | 3 + src/lift/aarch64/src/builder.rs | 9 +- src/lift/aarch64/src/int.rs | 2 +- src/lift/aarch64/src/lib.rs | 32 +- src/lift/aarch64/src/simd.rs | 38 +- src/lift/aarch64/src/tests.rs | 73 +- src/target/Cargo.toml | 1 + src/target/src/aarch64/disasm.rs | 1611 ----------------- src/target/src/aarch64/isel_tests.rs | 2 +- src/target/src/aarch64/listing.rs | 272 +++ src/target/src/aarch64/mod.rs | 4 +- 55 files changed, 8550 insertions(+), 3205 deletions(-) create mode 100644 src/disasm/Cargo.toml create mode 100644 src/disasm/examples/coverage.rs create mode 100644 src/disasm/examples/dump.rs create mode 100644 src/disasm/examples/fmtdiff.rs create mode 100644 src/disasm/src/analysis.rs create mode 100644 src/disasm/src/image.rs create mode 100644 src/disasm/src/lib.rs create mode 100644 src/isa/aarch64/Cargo.toml rename src/{target/src/aarch64 => isa/aarch64/src}/decode.rs (67%) create mode 100644 src/isa/aarch64/src/flow.rs create mode 100644 src/isa/aarch64/src/format/fp.rs create mode 100644 src/isa/aarch64/src/format/integer.rs create mode 100644 src/isa/aarch64/src/format/memory.rs create mode 100644 src/isa/aarch64/src/format/mod.rs create mode 100644 src/isa/aarch64/src/format/reg.rs create mode 100644 src/isa/aarch64/src/format/system.rs create mode 100644 src/isa/aarch64/src/format/vector.rs create mode 100644 src/isa/aarch64/src/fp.rs create mode 100644 src/isa/aarch64/src/gxf.rs create mode 100644 src/isa/aarch64/src/lib.rs create mode 100644 src/isa/aarch64/src/pauth.rs create mode 100644 src/isa/aarch64/src/simd.rs rename src/{target/src/aarch64 => isa/aarch64/src}/simd_struct.rs (100%) delete mode 100644 src/target/src/aarch64/disasm.rs create mode 100644 src/target/src/aarch64/listing.rs diff --git a/Cargo.lock b/Cargo.lock index 6113cd8..c74df12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,6 +172,14 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "volt-disasm" +version = "0.1.0" +dependencies = [ + "object", + "volt-isa-aarch64", +] + [[package]] name = "volt-emulator" version = "0.1.0" @@ -179,6 +187,7 @@ dependencies = [ "libc", "parking_lot", "volt-ir", + "volt-isa-aarch64", "volt-lift-aarch64", "volt-target", ] @@ -196,11 +205,16 @@ dependencies = [ name = "volt-ir" version = "0.1.0" +[[package]] +name = "volt-isa-aarch64" +version = "0.1.0" + [[package]] name = "volt-lift-aarch64" version = "0.1.0" dependencies = [ "volt-ir", + "volt-isa-aarch64", "volt-target", ] @@ -211,6 +225,7 @@ dependencies = [ "libc", "object", "volt-ir", + "volt-isa-aarch64", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 79238b9..e369bd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,8 @@ resolver = "3" members = [ "src/target", + "src/isa/aarch64", + "src/disasm", "src/ir", "src/emulator", "src/emulator/cli", @@ -15,6 +17,7 @@ license = "Apache-2.0" [workspace.dependencies] volt-ir = { path = "src/ir" } +volt-isa-aarch64 = { path = "src/isa/aarch64" } volt-target = { path = "src/target" } volt-emulator = { path = "src/emulator" } volt-lift-aarch64 = { path = "src/lift/aarch64" } diff --git a/README.md b/README.md index 25abf59..1467f3a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,25 @@ Independent Rust workspace for modular dynamic binary translation. +## Crates + +| Crate | Path | Role | +|---|---|---| +| `volt-isa-aarch64` | `src/isa/aarch64` | The AArch64 instruction set and nothing else: the single decoder (`decode`, delegating to the `fp`, `simd`, `pauth` and `gxf` families), control-flow classification (`flow`) and assembly text (`format`, `objdump` spelling). `no_std`; knows nothing of guest state, memory, blocks or executable memory. | +| `volt-ir` | `src/ir` | Typed SSA/CFG intermediate representation, verifier, legalization, bitcode. | +| `volt-target` | `src/target` | Native code generation (AArch64, x86-64), register allocation, object/link output, executable memory. | +| `volt-lift-aarch64` | `src/lift/aarch64` | AArch64 instruction semantics lifted to `volt-ir`, generic over an `Environment`. | +| `volt-emulator` | `src/emulator` | Machine models, translation-block policy (`aarch64::host::ends_block`), host execution of the instruction families the lifter does not implement, devices. | +| `volt-disasm` | `src/disasm` | Binary images, linear listing, control-flow recovery. | + +Every consumer of an instruction word decodes it once through `volt_isa_aarch64::decode` +and works from the result; nothing else in the workspace masks instruction bits. Where a +translator cuts its blocks is that translator's policy, not a property of an instruction. + +`cargo run --release -p volt-disasm --example fmtdiff -- [mnemonic] [limit]` checks +the formatter against GNU `objdump`, and `--example coverage -- ` reports which words +the decoder does not recognise. + ## AArch64 lifter `volt-lift-aarch64` (`src/lift/aarch64`) is a `no_std` crate that lifts decoded AArch64 diff --git a/src/disasm/Cargo.toml b/src/disasm/Cargo.toml new file mode 100644 index 0000000..b304d0e --- /dev/null +++ b/src/disasm/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "volt-disasm" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +path = "src/lib.rs" + +[dependencies] +object = { workspace = true, features = ["read_core", "elf", "macho", "std"] } +volt-isa-aarch64.workspace = true + +[lints.clippy] +too_many_arguments = "allow" diff --git a/src/disasm/examples/coverage.rs b/src/disasm/examples/coverage.rs new file mode 100644 index 0000000..f72f27c --- /dev/null +++ b/src/disasm/examples/coverage.rs @@ -0,0 +1,29 @@ +//! How much of an AArch64 image's executable sections the decoder recognises, and +//! which encodings it does not (the most frequent first, bucketed by opcode bits). +//! Formatting is checked separately by the `fmtdiff` example. +use std::collections::HashMap; +use volt_disasm::Image; +use volt_isa_aarch64::decode::decode; + +fn main() { + let path = std::env::args().nth(1).expect("usage: coverage "); + let img = Image::parse(&std::fs::read(path).unwrap()).unwrap(); + let (mut total, mut undecoded) = (0u64, 0u64); + let mut buckets: HashMap = HashMap::new(); + for s in img.segments.iter().filter(|s| s.exec) { + for c in s.data.chunks_exact(4) { + let w = u32::from_le_bytes(c.try_into().unwrap()); + total += 1; + if decode(w).is_err() { + undecoded += 1; + buckets.entry(w & 0xffe0_fc00).or_insert((0, w)).0 += 1; + } + } + } + println!("total {total} undecoded {undecoded}"); + let mut v: Vec<_> = buckets.into_iter().collect(); + v.sort_by_key(|x| std::cmp::Reverse(x.1.0)); + for (_, (n, w)) in v.iter().take(40) { + println!("{n:6} {w:08x}"); + } +} diff --git a/src/disasm/examples/dump.rs b/src/disasm/examples/dump.rs new file mode 100644 index 0000000..a8d740b --- /dev/null +++ b/src/disasm/examples/dump.rs @@ -0,0 +1,29 @@ +use volt_disasm::{Analysis, Image}; +fn main() { + let path = std::env::args() + .nth(1) + .expect("usage: dump [function]"); + let want = std::env::args().nth(2); + let img = Image::parse(&std::fs::read(path).unwrap()).unwrap(); + let a = Analysis::run(&img, &[]); + let blocks: usize = a.functions.values().map(|f| f.blocks.len()).sum(); + eprintln!( + "{} symbols, {} functions, {} blocks, {} xref targets", + img.symbols.len(), + a.functions.len(), + blocks, + a.xrefs.len() + ); + let Some(want) = want else { return }; + let f = a + .functions + .values() + .find(|f| f.name.as_deref() == Some(&want)) + .expect("no such function"); + for b in f.blocks.values() { + println!("block {:#x} -> {:x?} [{:?}]", b.start, b.succs, b.exit); + for i in img.disassemble(b.start, ((b.end - b.start) / 4) as usize) { + println!(" {:#x}: {:08x} {}", i.addr, i.word, i.text); + } + } +} diff --git a/src/disasm/examples/fmtdiff.rs b/src/disasm/examples/fmtdiff.rs new file mode 100644 index 0000000..651ffeb --- /dev/null +++ b/src/disasm/examples/fmtdiff.rs @@ -0,0 +1,158 @@ +//! Compare `volt_isa_aarch64::format` against GNU `objdump` over an AArch64 ELF. +//! +//! ```text +//! cargo run --release -p volt-disasm --example fmtdiff -- [mnemonic] [limit] +//! ``` +//! +//! For every instruction `objdump -d` lists it decodes the word with +//! `volt_isa_aarch64::decode`, renders it at its address, and compares the text +//! after normalising what the two spell differently on purpose (trailing `// ...` +//! comments, `` annotations, and the `0x` on branch targets). It +//! reports totals, then mismatches and undecodable words grouped by `objdump`'s +//! mnemonic, most frequent first, with one example of each. A mnemonic argument +//! restricts the report to that mnemonic and prints up to `limit` examples. + +use std::collections::HashMap; +use std::process::Command; +use volt_isa_aarch64::decode::decode; + +/// `objdump`'s own spelling, reduced to what must agree. +fn normalize(text: &str) -> String { + let text = text.split("//").next().unwrap_or(""); + let text = match text.find('<') { + Some(at) => &text[..at], + None => text, + }; + let text = text.split_whitespace().collect::>().join(" "); + let text = text.replace(", ", ","); + // A branch target is the last operand, written bare by objdump and `0x...` here. + match text.rsplit_once(',').or_else(|| text.rsplit_once(' ')) { + Some((head, last)) + if last.len() >= 4 + && !last.starts_with('#') + && last + .trim_start_matches("0x") + .chars() + .all(|c| c.is_ascii_hexdigit()) + && (last.starts_with("0x") || head.contains(' ') || !head.contains(',')) => + { + let sep = if text.contains(',') { "," } else { " " }; + format!("{head}{sep}{}", last.trim_start_matches("0x")) + } + _ => text, + } +} + +fn main() { + let mut args = std::env::args().skip(1); + let path = args + .next() + .expect("usage: fmtdiff [mnemonic] [limit]"); + let only = args.next(); + let limit: usize = args.next().map_or(3, |n| n.parse().unwrap()); + let listing = Command::new("objdump") + .args(["-d", "--no-show-raw-insn", "-w", &path]) + .output() + .expect("objdump"); + let raw = Command::new("objdump") + .args(["-d", "-w", &path]) + .output() + .expect("objdump"); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let raw = String::from_utf8_lossy(&raw.stdout).into_owned(); + + // address -> objdump text, and address -> word. + let mut text: Vec<(u64, String)> = Vec::new(); + for line in listing.lines() { + let Some((addr, rest)) = line.trim().split_once(":\t") else { + continue; + }; + if let Ok(addr) = u64::from_str_radix(addr, 16) { + text.push((addr, rest.trim().to_string())); + } + } + let mut words: HashMap = HashMap::new(); + for line in raw.lines() { + let mut parts = line.trim().splitn(3, '\t'); + let (Some(addr), Some(word)) = (parts.next(), parts.next()) else { + continue; + }; + if let (Ok(addr), Ok(word)) = ( + u64::from_str_radix(addr.trim_end_matches(':'), 16), + u32::from_str_radix(word.trim(), 16), + ) { + words.insert(addr, word); + } + } + + struct Group { + count: usize, + examples: Vec<(u32, String, String)>, + } + let mut mismatch: HashMap = HashMap::new(); + let mut undecoded: HashMap = HashMap::new(); + let (mut total, mut matched) = (0usize, 0usize); + for (addr, expected) in &text { + let Some(&word) = words.get(addr) else { + continue; + }; + // Data in the text section: `.word`, `.inst`, `udf`. + let mnemonic = expected.split_whitespace().next().unwrap_or("").to_string(); + if mnemonic.starts_with('.') { + continue; + } + total += 1; + let (group, ours) = match decode(word) { + Ok(instruction) => { + let ours = instruction.display_at(*addr).to_string(); + if normalize(&ours) == normalize(expected) { + matched += 1; + continue; + } + (&mut mismatch, ours) + } + Err(_) => (&mut undecoded, String::new()), + }; + if only.as_ref().is_some_and(|m| *m != mnemonic) { + let _ = ours; + group + .entry(mnemonic) + .or_insert(Group { + count: 0, + examples: Vec::new(), + }) + .count += 1; + continue; + } + let entry = group.entry(mnemonic).or_insert(Group { + count: 0, + examples: Vec::new(), + }); + entry.count += 1; + if entry.examples.len() < limit { + entry.examples.push((word, expected.clone(), ours)); + } + } + println!( + "total {total} matched {matched} mismatched {} undecoded {}", + mismatch.values().map(|g| g.count).sum::(), + undecoded.values().map(|g| g.count).sum::() + ); + for (title, groups) in [("MISMATCH", &mismatch), ("UNDECODED", &undecoded)] { + let mut groups: Vec<_> = groups.iter().collect(); + groups.sort_by_key(|(name, g)| (std::cmp::Reverse(g.count), (*name).clone())); + for (name, group) in groups + .into_iter() + .filter(|(name, _)| only.as_ref().is_none_or(|m| m == *name)) + .take(if only.is_some() { usize::MAX } else { 40 }) + { + println!("{title} {:>6} {name}", group.count); + for (word, expected, ours) in &group.examples { + println!(" {word:08x} objdump: {expected}"); + if title == "MISMATCH" { + println!(" ours: {ours}"); + } + } + } + } +} diff --git a/src/disasm/src/analysis.rs b/src/disasm/src/analysis.rs new file mode 100644 index 0000000..1b00130 --- /dev/null +++ b/src/disasm/src/analysis.rs @@ -0,0 +1,178 @@ +//! Recursive-descent function discovery, basic blocks and cross-references. +use crate::Image; +use std::collections::{BTreeMap, BTreeSet}; +use volt_isa_aarch64::flow::{Flow, flow}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Block { + pub start: u64, + /// Address one past the last instruction. + pub end: u64, + /// Successor block starts within the same function. + pub succs: Vec, + /// How the block ends. + pub exit: Flow, +} + +#[derive(Clone, Debug)] +pub struct Function { + pub entry: u64, + pub name: Option, + pub blocks: BTreeMap, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum XrefKind { + Call, + Jump, + CondJump, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Xref { + pub from: u64, + pub kind: XrefKind, +} + +#[derive(Debug, Default)] +pub struct Analysis { + pub functions: BTreeMap, + /// Target address -> references to it, sorted. + pub xrefs: BTreeMap>, +} + +impl Analysis { + /// Discover functions from function symbols, the entry point and `extra` + /// roots, following direct calls. + pub fn run(image: &Image, extra: &[u64]) -> Self { + let mut roots: BTreeSet = image + .symbols + .iter() + .filter(|s| s.func && image.is_exec(s.addr)) + .map(|s| s.addr) + .collect(); + roots.extend(image.entry.filter(|&e| image.is_exec(e))); + roots.extend(extra.iter().copied().filter(|&a| image.is_exec(a))); + let known = roots.clone(); + + let mut out = Self::default(); + let mut queue: Vec = roots.into_iter().collect(); + while let Some(entry) = queue.pop() { + if out.functions.contains_key(&entry) { + continue; + } + let f = out.build(image, entry, &known); + for b in f.blocks.values() { + if let Flow::Call(t) = b.exit + && image.is_exec(t) + { + queue.push(t); + } + } + out.functions.insert(entry, f); + } + for v in out.xrefs.values_mut() { + v.sort(); + v.dedup(); + } + out + } + + pub fn function_containing(&self, addr: u64) -> Option<&Function> { + self.functions.values().find(|f| { + f.blocks + .range(..=addr) + .next_back() + .is_some_and(|(_, b)| addr < b.end) + }) + } + + fn build(&mut self, image: &Image, entry: u64, known: &BTreeSet) -> Function { + // Pass 1: reachable instructions and leaders. + let mut insns: BTreeMap = BTreeMap::new(); + let mut leaders = BTreeSet::from([entry]); + let mut work = vec![entry]; + while let Some(mut pc) = work.pop() { + loop { + if insns.contains_key(&pc) || !image.is_exec(pc) { + break; + } + let Some(word) = image.read_u32(pc) else { + break; + }; + let fl = flow(word, pc); + insns.insert(pc, fl); + let kind = match fl { + Flow::Call(_) => Some(XrefKind::Call), + Flow::Jump(_) => Some(XrefKind::Jump), + Flow::CondJump(_) => Some(XrefKind::CondJump), + _ => None, + }; + if let (Some(kind), Some(t)) = (kind, fl.target()) { + self.xrefs + .entry(t) + .or_default() + .push(Xref { from: pc, kind }); + } + match fl { + // A jump to another function's entry is a tail call. + Flow::Jump(t) | Flow::CondJump(t) if !(known.contains(&t) && t != entry) => { + leaders.insert(t); + work.push(t); + } + _ => {} + } + if fl.ends_block() { + if fl.falls_through() { + leaders.insert(pc + 4); + work.push(pc + 4); + } + break; + } + pc += 4; + } + } + // Pass 2: cut blocks at leaders. + let mut blocks = BTreeMap::new(); + for &start in leaders.iter().filter(|l| insns.contains_key(l)) { + let mut pc = start; + let (end, exit) = loop { + let fl = insns[&pc]; + let next = pc + 4; + if fl.ends_block() || leaders.contains(&next) || !insns.contains_key(&next) { + break (next, fl); + } + pc = next; + }; + let mut succs = Vec::new(); + if exit.falls_through() && insns.contains_key(&end) { + succs.push(end); + } + if let Flow::Jump(t) | Flow::CondJump(t) = exit + && insns.contains_key(&t) + && !(known.contains(&t) && t != entry) + { + succs.push(t); + } + blocks.insert( + start, + Block { + start, + end, + succs, + exit, + }, + ); + } + let name = image + .symbols + .iter() + .find(|s| s.addr == entry && s.func) + .map(|s| s.name.clone()); + Function { + entry, + name, + blocks, + } + } +} diff --git a/src/disasm/src/image.rs b/src/disasm/src/image.rs new file mode 100644 index 0000000..e55ea2e --- /dev/null +++ b/src/disasm/src/image.rs @@ -0,0 +1,134 @@ +//! A loaded binary: mapped segments and symbols. +use object::{Object, ObjectSection, ObjectSymbol, SectionKind, SymbolKind}; + +#[derive(Clone, Debug)] +pub struct Segment { + pub name: String, + pub addr: u64, + pub data: Vec, + pub exec: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Symbol { + pub name: String, + pub addr: u64, + pub size: u64, + pub func: bool, +} + +#[derive(Clone, Debug, Default)] +pub struct Image { + /// Sorted by address. + pub segments: Vec, + /// Sorted by address. + pub symbols: Vec, + pub entry: Option, +} + +impl Image { + /// Parse an ELF or Mach-O image. Raw files are not detected; use + /// [`Image::raw`]. + pub fn parse(bytes: &[u8]) -> Result { + let file = object::File::parse(bytes).map_err(|e| e.to_string())?; + if file.architecture() != object::Architecture::Aarch64 { + return Err(format!( + "unsupported architecture {:?}", + file.architecture() + )); + } + let mut segments = Vec::new(); + for s in file.sections() { + let exec = s.kind() == SectionKind::Text; + if !(exec + || matches!( + s.kind(), + SectionKind::Data | SectionKind::ReadOnlyData | SectionKind::ReadOnlyString + )) + { + continue; + } + let Ok(data) = s.data() else { continue }; + if data.is_empty() { + continue; + } + segments.push(Segment { + name: s.name().unwrap_or("").to_string(), + addr: s.address(), + data: data.to_vec(), + exec, + }); + } + segments.sort_by_key(|s| s.addr); + let mut symbols: Vec = file + .symbols() + .chain(file.dynamic_symbols()) + .filter(|s| s.address() != 0 && !s.name().unwrap_or("").is_empty()) + .filter(|s| { + matches!( + s.kind(), + SymbolKind::Text | SymbolKind::Data | SymbolKind::Unknown + ) + }) + .map(|s| Symbol { + name: s.name().unwrap_or("").to_string(), + addr: s.address(), + size: s.size(), + func: s.kind() == SymbolKind::Text, + }) + .collect(); + symbols.sort_by(|a, b| a.addr.cmp(&b.addr).then_with(|| a.name.cmp(&b.name))); + symbols.dedup(); + let entry = Some(file.entry()).filter(|&e| e != 0); + Ok(Self { + segments, + symbols, + entry, + }) + } + + /// An image of one executable blob at `addr`. + pub fn raw(addr: u64, code: Vec) -> Self { + Self { + segments: vec![Segment { + name: "raw".into(), + addr, + data: code, + exec: true, + }], + symbols: Vec::new(), + entry: Some(addr), + } + } + + fn segment_at(&self, addr: u64) -> Option<&Segment> { + let i = self + .segments + .partition_point(|s| s.addr <= addr) + .checked_sub(1)?; + let s = &self.segments[i]; + (addr - s.addr < s.data.len() as u64).then_some(s) + } + + pub fn is_exec(&self, addr: u64) -> bool { + self.segment_at(addr).is_some_and(|s| s.exec) + } + + pub fn read_u32(&self, addr: u64) -> Option { + let s = self.segment_at(addr)?; + let o = (addr - s.addr) as usize; + Some(u32::from_le_bytes(s.data.get(o..o + 4)?.try_into().ok()?)) + } + + /// The symbol containing `addr` (or the nearest preceding one when sizes + /// are unknown), with the offset into it. + pub fn symbolize(&self, addr: u64) -> Option<(&Symbol, u64)> { + let i = self + .symbols + .partition_point(|s| s.addr <= addr) + .checked_sub(1)?; + let s = &self.symbols[i]; + let off = addr - s.addr; + (s.size == 0 || off < s.size).then_some((s, off)) + } +} diff --git a/src/disasm/src/lib.rs b/src/disasm/src/lib.rs new file mode 100644 index 0000000..d1d31d7 --- /dev/null +++ b/src/disasm/src/lib.rs @@ -0,0 +1,62 @@ +//! Disassembly framework: binary images, linear listing, and control-flow +//! recovery over `volt-isa-aarch64`'s decoder and formatter. +pub mod analysis; +pub mod image; + +pub use analysis::{Analysis, Block, Function, Xref, XrefKind}; +pub use image::{Image, Segment, Symbol}; + +use volt_isa_aarch64::decode::decode; +use volt_isa_aarch64::flow::{Flow, flow}; +use volt_isa_aarch64::format::{self, Symbols}; + +/// One decoded instruction. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Insn { + pub addr: u64, + pub word: u32, + pub text: String, + pub flow: Flow, +} + +impl Insn { + /// Decode `word` at `addr`; targets print absolute, annotated by `symbols`. + pub fn decode(addr: u64, word: u32, symbols: &dyn Symbols) -> Self { + // One decode serves both the text and the flow; a word the decoder does not + // recognise prints raw and is classified by `flow`. + match decode(word) { + Ok(instruction) => Self { + addr, + word, + text: instruction.display_with(addr, symbols).to_string(), + flow: instruction.flow(addr), + }, + Err(_) => Self { + addr, + word, + text: format::word_with(word, addr, symbols).to_string(), + flow: flow(word, addr), + }, + } + } +} + +impl Symbols for Image { + fn lookup(&self, addr: u64) -> Option<(&str, u64)> { + self.symbolize(addr) + .map(|(s, offset)| (s.name.as_str(), offset)) + } +} + +impl Image { + /// Decode `count` instructions starting at `addr`, stopping at the end of + /// the mapped segment. + pub fn disassemble(&self, addr: u64, count: usize) -> Vec { + (0..count as u64) + .map_while(|i| { + let a = addr + i * 4; + self.read_u32(a).map(|w| Insn::decode(a, w, self)) + }) + .collect() + } +} diff --git a/src/emulator/Cargo.toml b/src/emulator/Cargo.toml index d48a800..cc8e9c3 100644 --- a/src/emulator/Cargo.toml +++ b/src/emulator/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true [dependencies] volt-ir.workspace = true +volt-isa-aarch64.workspace = true volt-target.workspace = true volt-lift-aarch64.workspace = true libc = { workspace = true, optional = true } diff --git a/src/emulator/examples/jit_bench.rs b/src/emulator/examples/jit_bench.rs index 1a581a7..413151b 100644 --- a/src/emulator/examples/jit_bench.rs +++ b/src/emulator/examples/jit_bench.rs @@ -20,14 +20,14 @@ fn block(n: usize, seed: u32) -> Vec { fn main() { dispatch(); { - use volt_emulator::aarch64::host::dispatch; + use volt_emulator::aarch64::host::ends_block; + use volt_isa_aarch64::decode::decode; let b = block(63, 1); let n = b .chunks_exact(4) .take_while(|c| { - !dispatch(u32::from_le_bytes((*c).try_into().unwrap())) - .unwrap() - .terminates() + let word = u32::from_le_bytes((*c).try_into().unwrap()); + !ends_block(&decode(word).unwrap(), false) }) .count(); println!("non-terminating prefix of 63-block: {n}"); diff --git a/src/emulator/src/aarch64/ahead.rs b/src/emulator/src/aarch64/ahead.rs index d07c453..f4c97d0 100644 --- a/src/emulator/src/aarch64/ahead.rs +++ b/src/emulator/src/aarch64/ahead.rs @@ -19,13 +19,16 @@ //! on a worker thread. use super::cache::{MAX_INSNS, SharedBlocks}; -use super::host::{Dispatch, dispatch}; +use super::host::ends_block; use alloc::collections::VecDeque; use alloc::sync::{Arc, Weak}; use std::collections::HashSet; use std::sync::{Condvar, Mutex, MutexGuard, PoisonError}; use std::thread; -use volt_target::aarch64::decode::Instruction; +use volt_isa_aarch64::{ + decode::{Instruction, decode}, + flow::Flow, +}; /// Reads guest-physical bytes without any vCPU state, for worker threads. pub trait CodeReader: Send + Sync { @@ -67,25 +70,15 @@ fn lock(state: &Mutex) -> MutexGuard<'_, State> { /// Where control can go after `last`, a block's final instruction at `last_pc`. /// Fixed targets only: an indirect branch or return has none to offer. -fn successors(last: Dispatch, last_pc: u64) -> ([u64; 2], usize) { +fn successors(instruction: Instruction, last_pc: u64) -> ([u64; 2], usize) { let next = last_pc.wrapping_add(4); - let at = |offset: i64| last_pc.wrapping_add(offset as u64); - let instruction = match last { - Dispatch::Guest(instruction) => instruction, - // A host word falls through, as a block end that does not branch does. - Dispatch::Host(_) => return ([next, 0], 1), - }; - match instruction { - Instruction::B(offset) => ([at(offset), 0], 1), - Instruction::Call(call) if call.link => ([at(call.target), next], 2), - Instruction::Call(call) => ([at(call.target), 0], 1), - Instruction::BCond(branch) => ([at(branch.offset), next], 2), - Instruction::TestBranch(test) => ([at(test.offset), next], 2), - Instruction::Indirect(branch) if branch.link => ([next, 0], 1), - Instruction::Indirect(_) | Instruction::Eret => ([0, 0], 0), - // Every other block end, a trap or a cut at the instruction limit, resumes - // at the next instruction. - _ => ([next, 0], 1), + match instruction.flow(last_pc) { + Flow::Jump(target) => ([target, 0], 1), + Flow::CondJump(target) | Flow::Call(target) => ([target, next], 2), + Flow::JumpIndirect | Flow::Return => ([0, 0], 0), + // Every other block end, a call returning, a trap or a cut at the + // instruction limit, resumes at the next instruction. + Flow::Next | Flow::CallIndirect | Flow::Stop | Flow::Unrecognized => ([next, 0], 1), } } @@ -119,7 +112,7 @@ impl Ahead { /// Offer the blocks that can follow `last` (the final instruction of the /// block just built, at `last_pc`). `physical` is the physical address of /// `last_pc`; only successors on its page are taken. - pub fn offer_successors(&self, last: Dispatch, last_pc: u64, physical: u64) { + pub fn offer_successors(&self, last: Instruction, last_pc: u64, physical: u64) { let (targets, count) = successors(last, last_pc); self.offer(&targets[..count], last_pc, physical, 0); } @@ -188,12 +181,12 @@ impl Ahead { return; } let word = u32::from_le_bytes(page[count..count + 4].try_into().unwrap()); - let Ok(instruction) = dispatch(word) else { + let Ok(instruction) = decode(word) else { return; }; last = Some((instruction, job.pc + count as u64)); count += 4; - if instruction.terminates_with(shared.inline_memory()) + if ends_block(&instruction, shared.inline_memory()) || n + 1 == MAX_INSNS || (job.pc + count as u64) & 0xfff == 0 { @@ -265,7 +258,7 @@ mod tests { #[test] fn fixed_targets_follow_each_kind_of_block_end() { - use volt_target::aarch64::decode::{Call, Test, Width}; + use volt_isa_aarch64::decode::{Call, Indirect, IndirectKind, Test, Width}; let at = 0x1000; let test = Instruction::TestBranch(Test { rt: 0, @@ -281,14 +274,17 @@ mod tests { (call(false), vec![0x1040]), (test, vec![0x0ff8, 0x1004]), ( - Instruction::Indirect(volt_target::aarch64::decode::Indirect { + Instruction::Indirect(Indirect { rn: 1, - link: false, + kind: IndirectKind::Jump, }), vec![], ), ( - Instruction::Indirect(volt_target::aarch64::decode::Indirect { rn: 1, link: true }), + Instruction::Indirect(Indirect { + rn: 1, + kind: IndirectKind::Call, + }), vec![0x1004], ), (Instruction::Eret, vec![]), @@ -296,7 +292,7 @@ mod tests { (Instruction::Nop, vec![0x1004]), ]; for (last, expected) in cases { - let (targets, n) = successors(Dispatch::Guest(last), at); + let (targets, n) = successors(last, at); assert_eq!(&targets[..n], expected.as_slice(), "{last:?}"); } } @@ -313,14 +309,14 @@ mod tests { ]); let shared = Arc::new(SharedBlocks::new(false)); let ahead = Ahead::start(&shared, memory, 2); - ahead.offer_successors(Dispatch::Guest(Instruction::B(0x10)), 0x1004, 0x1004); + ahead.offer_successors(Instruction::B(0x10), 0x1004, 0x1004); wait_for(&shared, 2); // The block after the branch, and the one after its `svc`: nothing more, // even though the last one branches to itself. thread::sleep(Duration::from_millis(50)); assert_eq!(shared.compiled(), 2); // Offering again builds nothing new. - ahead.offer_successors(Dispatch::Guest(Instruction::B(0x10)), 0x1004, 0x1004); + ahead.offer_successors(Instruction::B(0x10), 0x1004, 0x1004); thread::sleep(Duration::from_millis(50)); assert_eq!(shared.compiled(), 2); // The vCPU's own lookup of the same bytes finds the block already built. @@ -337,7 +333,7 @@ mod tests { let shared = Arc::new(SharedBlocks::new(false)); let ahead = Ahead::start(&shared, memory, 1); // A branch from page 1 to page 2 must not be read on this page's translation. - ahead.offer_successors(Dispatch::Guest(Instruction::B(0x1004)), 0x1ffc, 0x1ffc); + ahead.offer_successors(Instruction::B(0x1004), 0x1ffc, 0x1ffc); thread::sleep(Duration::from_millis(50)); assert_eq!(shared.compiled(), 0); ahead.shutdown(); diff --git a/src/emulator/src/aarch64/cache.rs b/src/emulator/src/aarch64/cache.rs index 65dd573..a93e0f9 100644 --- a/src/emulator/src/aarch64/cache.rs +++ b/src/emulator/src/aarch64/cache.rs @@ -2,7 +2,7 @@ use super::{ ahead::{Ahead, CodeReader}, compile::{Block, Error, compile_with}, cpu::Cpu, - host::dispatch, + host::ends_block, }; use core::hash::{Hash, Hasher}; use core::sync::atomic::{AtomicUsize, Ordering}; @@ -356,7 +356,7 @@ impl Cache { } count += 4; let word = u32::from_le_bytes(room.try_into().unwrap()); - let instruction = match dispatch(word) { + let instruction = match volt_isa_aarch64::decode::decode(word) { Ok(insn) => insn, // Run the decoded prefix first: the undecodable word then heads the // next block, where it is reported at its own address. @@ -366,14 +366,14 @@ impl Cache { } Err(_) => { // Architecturally undefined words are the guest's to handle. - if !volt_target::aarch64::decode::is_undefined(word) { + if !volt_isa_aarch64::decode::is_undefined(word) { eprintln!("cache decode failure at pc {at:x}: word {word:08x}"); } return Err(Error::Decode { word, pc: at }); } }; last = Some((instruction, at)); - if instruction.terminates_with(self.shared.inline_memory()) { + if ends_block(&instruction, self.shared.inline_memory()) { complete = true; break; } diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index 98dbc99..1e7f56e 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -1,10 +1,11 @@ use super::{ cpu::{Cpu, Trap}, environment::CpuEnvironment, - host::{Dispatch, dispatch}, + host::{ends_block, runs_on_host}, }; use crate::memory::MemoryError; use core::fmt; +use volt_isa_aarch64::decode::decode; use volt_lift_aarch64::{LiftError, Lifter}; use volt_target::native; @@ -82,7 +83,7 @@ pub fn compile(guest_pc: u64, bytes: &[u8]) -> Result { /// Compile a block. With `inline_memory`, plain loads and stores get an inline /// data-TLB fast path and do not end the block; the caller's block scan must -/// use `Instruction::terminates_with(true)` to match. +/// use `host::ends_block(_, true)` to match. pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result { if bytes.is_empty() { return Err(Error::EmptyBlock); @@ -95,17 +96,17 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< let mut ended = false; for chunk in bytes.chunks_exact(4) { let word = u32::from_le_bytes(chunk.try_into().unwrap()); - match dispatch(word) { - Ok(Dispatch::Guest(instruction)) => { - ended = instruction.terminates_with(inline_memory); - lifter.lift(pc, instruction)?; - } - Ok(Dispatch::Host(host_word)) => { + match decode(word) { + Ok(instruction) if runs_on_host(&instruction) => { // The host runs this word (see `host::run`), and it ends the block. let (builder, env) = lifter.parts(); - env.host(builder, pc, host_word); + env.host(builder, pc, word); ended = true; } + Ok(instruction) => { + ended = ends_block(&instruction, inline_memory); + lifter.lift(pc, instruction)?; + } Err(_) => { eprintln!("compile decode failure at pc {pc:x}: word {word:08x}, full block:"); for (i, c) in bytes.chunks_exact(4).enumerate() { diff --git a/src/emulator/src/aarch64/cpu.rs b/src/emulator/src/aarch64/cpu.rs index 6a2d32b..42aee5f 100644 --- a/src/emulator/src/aarch64/cpu.rs +++ b/src/emulator/src/aarch64/cpu.rs @@ -1,5 +1,5 @@ -use volt_target::aarch64::decode::{AtomicKind, SignExtend}; -use volt_target::aarch64::simd_struct::StructDesc; +use volt_isa_aarch64::decode::{AtomicKind, SignExtend}; +use volt_isa_aarch64::simd_struct::StructDesc; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] @@ -159,13 +159,13 @@ pub struct Cpu { pub apple_bank: ImpDefBank, } -/// `S3__c15` registers, indexed by `volt_target::aarch64::decode::apple_bank_slot`. +/// `S3__c15` registers, indexed by `volt_isa_aarch64::decode::apple_bank_slot`. #[derive(Clone, Debug)] -pub struct ImpDefBank(pub [u64; volt_target::aarch64::decode::APPLE_BANK_SLOTS]); +pub struct ImpDefBank(pub [u64; volt_isa_aarch64::decode::APPLE_BANK_SLOTS]); impl Default for ImpDefBank { fn default() -> Self { - Self([0; volt_target::aarch64::decode::APPLE_BANK_SLOTS]) + Self([0; volt_isa_aarch64::decode::APPLE_BANK_SLOTS]) } } @@ -236,7 +236,7 @@ pub struct System { pub tpidr_el2: u64, pub cntvoff_el2: u64, /// Apple IMP-DEF configuration registers, indexed by `SystemRegister::Apple`. - pub apple: [u64; volt_target::aarch64::decode::APPLE_SLOTS], + pub apple: [u64; volt_isa_aarch64::decode::APPLE_SLOTS], /// `MIDR_EL1`, `ID_AA64PFR0_EL1` and `ID_AA64MMFR0_EL1`: the identity this CPU /// advertises, set by the board. pub midr_el1: u64, @@ -306,7 +306,7 @@ impl Default for System { cntp_ctl_el0: 0, cntp_cval_el0: 0, actlr_el1: 0, - apple: [0; volt_target::aarch64::decode::APPLE_SLOTS], + apple: [0; volt_isa_aarch64::decode::APPLE_SLOTS], midr_el1: crate::aarch64::identification::DEFAULT_MIDR_EL1, id_aa64pfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64PFR0_EL1, id_aa64mmfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64MMFR0_EL1, diff --git a/src/emulator/src/aarch64/environment.rs b/src/emulator/src/aarch64/environment.rs index c98c735..f87ee18 100644 --- a/src/emulator/src/aarch64/environment.rs +++ b/src/emulator/src/aarch64/environment.rs @@ -4,13 +4,13 @@ use super::cpu::{Cpu, DTLB_ENTRIES, Exclusive as CpuExclusive, System as CpuSystem, Trap}; use core::mem::offset_of; use volt_ir::function::{BinOp as B, CmpOp as C, Value}; +use volt_isa_aarch64::decode::*; use volt_lift_aarch64::{Builder, Environment, LiftError, MemoryAccess, Writeback}; -use volt_target::aarch64::decode::*; /// Lowers guest instructions against [`Cpu`]: the lifter's machine policy. pub(super) struct CpuEnvironment { /// Plain loads and stores get an inline data-TLB fast path and do not end - /// the block; the block scan must use `Instruction::terminates_with(true)`. + /// the block; the block scan must use `host::ends_block(_, true)`. inline_memory: bool, /// Guest instructions started but not yet added to `cntvct_el0`. The /// counter is bumped once per exit path (`exit`) and before anything that @@ -265,6 +265,8 @@ impl CpuEnvironment { None => {} } } + // `a.ordered` (`ldar`, `stlr`, `ldapr`, ...) and the unscaled/unprivileged offset forms + // run as the plain access: the machines model no memory ordering or EL0 permission check. fn scalar(&self, b: &Builder, pc: u64, a: Memory, address: Value, post: Option) { let trap = if a.op == MemoryOp::Store { Trap::Store @@ -293,15 +295,15 @@ impl CpuEnvironment { a.size, MemoryOp::Load, a.rt, - SignExtend::None, + a.signed, None, |env| { - env.setup_memory(b, address, a.size, a.rt, SignExtend::None, MemoryOp::Load); + env.setup_memory(b, address, a.size, a.rt, a.signed, MemoryOp::Load); env.trap(b, pc.wrapping_add(4), Trap::Load); }, ); } else { - self.setup_memory(b, address, a.size, a.rt, SignExtend::None, MemoryOp::Load); + self.setup_memory(b, address, a.size, a.rt, a.signed, MemoryOp::Load); self.trap(b, pc.wrapping_add(4), Trap::Load); } } @@ -401,6 +403,7 @@ impl CpuEnvironment { b.set_block(resume); } } + // `a.access` (`ldr q` vs one-register `ld1`) and unscaled offsets move the same bytes. fn simd_memory( &self, b: &Builder, @@ -470,6 +473,7 @@ impl CpuEnvironment { b.set_block(resume); } } + // `a.order` (`ldaxr`/`stlxr`) is ignored: the monitor and host accesses already order. fn exclusive(&self, b: &Builder, pc: u64, a: Exclusive) { self.setup_memory( b, @@ -500,6 +504,8 @@ impl CpuEnvironment { }, ); } + // `a.order` (acquire/release) is ignored: every host read-modify-write is sequentially + // consistent. fn atomic(&self, b: &Builder, pc: u64, a: Atomic) { // Read-modify-write needs a host atomic, so it always takes the machine // slow path. `dests` receive the words read (31 discards) and `operands` @@ -531,7 +537,7 @@ impl CpuEnvironment { self.trap(b, pc.wrapping_add(4), Trap::Atomic); } fn simd_struct(&self, b: &Builder, pc: u64, a: SimdStruct) { - use volt_target::aarch64::simd_struct::StructDesc; + use volt_isa_aarch64::simd_struct::StructDesc; // Element-wise accesses with lane merging need the machine slow path. let d = a.desc; let address = b.reg(b.i64, a.rn, false); @@ -654,7 +660,7 @@ impl CpuEnvironment { b.store(pan, b.imm(b.i64, B::BitAnd, bit, 1)); } } - RazWi | AppleZero => { + RazWi { .. } | AppleZero { .. } => { if a.read { b.put(a.rt, b.k(b.i64, 0)); } diff --git a/src/emulator/src/aarch64/fp.rs b/src/emulator/src/aarch64/fp.rs index d3224a0..1f506e1 100644 --- a/src/emulator/src/aarch64/fp.rs +++ b/src/emulator/src/aarch64/fp.rs @@ -1,7 +1,7 @@ //! Scalar floating point (single and double precision) and the vector `fabs`/`fneg` //! forms, executed with host `f32`/`f64` arithmetic. //! -//! Decode only classifies a word with [`parse`]; the JIT stores the word in +//! Decoding lives in `volt_isa_aarch64::fp`; the JIT stores the word in //! `Cpu::value` and traps, and the machine (or user mode) runs [`run`] on the CPU. //! Results follow the Arm rules where they differ from the host's: an invalid //! operation yields the default NaN (positive, quiet), a NaN operand propagates @@ -11,332 +11,8 @@ //! rounding is always to nearest, ties to even), and the `FPSR` exception flags. use super::Cpu; -use volt_target::aarch64::decode::expand_immediate; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Precision { - Single, - Double, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Unary { - Move, - Abs, - Neg, - Sqrt, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Binary { - Mul, - Div, - Add, - Sub, - Max, - Min, - MaxNum, - MinNum, - Nmul, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Fused { - Madd, - Msub, - Nmadd, - Nmsub, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Op { - Unary { - kind: Unary, - p: Precision, - rd: u8, - rn: u8, - }, - /// `fcvt`: between single and double; `p` is the source precision. - Convert { - p: Precision, - rd: u8, - rn: u8, - }, - Binary { - kind: Binary, - p: Precision, - rd: u8, - rn: u8, - rm: u8, - }, - Fused { - kind: Fused, - p: Precision, - rd: u8, - rn: u8, - rm: u8, - ra: u8, - }, - /// `fcmp`/`fcmpe`; no `rm` compares with zero. - Compare { - p: Precision, - rn: u8, - rm: Option, - }, - CondCompare { - p: Precision, - rn: u8, - rm: u8, - cond: u32, - nzcv: u32, - }, - Select { - p: Precision, - rd: u8, - rn: u8, - rm: u8, - cond: u32, - }, - Immediate { - p: Precision, - rd: u8, - imm8: u64, - }, - /// `scvtf`/`ucvtf` from a general register. - IntToFloat { - signed: bool, - p: Precision, - wide: bool, - rd: u8, - rn: u8, - }, - /// `fcvtzs`/`fcvtzu` to a general register. - FloatToInt { - signed: bool, - p: Precision, - wide: bool, - rd: u8, - rn: u8, - }, - /// `scvtf`/`ucvtf` between SIMD&FP registers (the source holds an integer). - ScalarIntToFloat { - signed: bool, - p: Precision, - rd: u8, - rn: u8, - }, - /// `fcvtzs`/`fcvtzu` between SIMD&FP registers. - ScalarFloatToInt { - signed: bool, - p: Precision, - rd: u8, - rn: u8, - }, - /// Vector `fabs`/`fneg`. - VectorSign { - negate: bool, - double: bool, - q: bool, - rd: u8, - rn: u8, - }, -} - -fn precision(ty: u32) -> Option { - match ty { - 0 => Some(Precision::Single), - 1 => Some(Precision::Double), - // 3 is half precision, which is not modelled; 2 is reserved. - _ => None, - } -} - -fn parse(word: u32) -> Option { - let rd = (word & 31) as u8; - let rn = ((word >> 5) & 31) as u8; - let rm = ((word >> 16) & 31) as u8; - let ty = (word >> 22) & 3; - // Scalar floating point: `M=0 S=0 01111 0 type 1 ...`. - if word & 0xff20_0c00 == 0x1e20_0800 { - let kind = match (word >> 12) & 15 { - 0 => Binary::Mul, - 1 => Binary::Div, - 2 => Binary::Add, - 3 => Binary::Sub, - 4 => Binary::Max, - 5 => Binary::Min, - 6 => Binary::MaxNum, - 7 => Binary::MinNum, - 8 => Binary::Nmul, - _ => return None, - }; - return Some(Op::Binary { - kind, - p: precision(ty)?, - rd, - rn, - rm, - }); - } - // One-source: bits 14:10 = 10000; the opcode (bits 20:15) includes bit 15. - if word & 0xff20_7c00 == 0x1e20_4000 { - let p = precision(ty)?; - let kind = match (word >> 15) & 0x3f { - 0 => Unary::Move, - 1 => Unary::Abs, - 2 => Unary::Neg, - 3 => Unary::Sqrt, - // `fcvt` to single from double, to double from single. - 4 if p == Precision::Double => return Some(Op::Convert { p, rd, rn }), - 5 if p == Precision::Single => return Some(Op::Convert { p, rd, rn }), - _ => return None, - }; - return Some(Op::Unary { kind, p, rd, rn }); - } - if word & 0xff20_fc07 == 0x1e20_2000 { - let zero = word & 8 != 0; - return Some(Op::Compare { - p: precision(ty)?, - rn, - rm: (!zero).then_some(rm), - }); - } - if word & 0xff20_0c10 == 0x1e20_0400 { - return Some(Op::CondCompare { - p: precision(ty)?, - rn, - rm, - cond: (word >> 12) & 15, - nzcv: word & 15, - }); - } - if word & 0xff20_0c00 == 0x1e20_0c00 { - return Some(Op::Select { - p: precision(ty)?, - rd, - rn, - rm, - cond: (word >> 12) & 15, - }); - } - if word & 0xff20_1fe0 == 0x1e20_1000 { - return Some(Op::Immediate { - p: precision(ty)?, - rd, - imm8: u64::from((word >> 13) & 0xff), - }); - } - if word & 0xff00_0000 == 0x1f00_0000 { - let kind = match ((word >> 21) & 1, (word >> 15) & 1) { - (0, 0) => Fused::Madd, - (0, _) => Fused::Msub, - (_, 0) => Fused::Nmadd, - _ => Fused::Nmsub, - }; - return Some(Op::Fused { - kind, - p: precision(ty)?, - rd, - rn, - rm, - ra: ((word >> 10) & 31) as u8, - }); - } - // Conversions between floating point and general registers. The `fmov` forms - // (rmode 0, opcode 6 and 7) are `SimdFmov`. - if word & 0x7f20_fc00 == 0x1e20_0000 { - let (p, wide) = (precision(ty)?, word & 0x8000_0000 != 0); - return match ((word >> 19) & 3, (word >> 16) & 7) { - (0, 2) => Some(Op::IntToFloat { - signed: true, - p, - wide, - rd, - rn, - }), - (0, 3) => Some(Op::IntToFloat { - signed: false, - p, - wide, - rd, - rn, - }), - (3, 0) => Some(Op::FloatToInt { - signed: true, - p, - wide, - rd, - rn, - }), - (3, 1) => Some(Op::FloatToInt { - signed: false, - p, - wide, - rd, - rn, - }), - _ => None, - }; - } - // The same conversions between SIMD&FP registers: bit 22 is `sz`. - let p = if word & 0x0040_0000 != 0 { - Precision::Double - } else { - Precision::Single - }; - match word & 0xffbf_fc00 { - 0x5e21_d800 => { - return Some(Op::ScalarIntToFloat { - signed: true, - p, - rd, - rn, - }); - } - 0x7e21_d800 => { - return Some(Op::ScalarIntToFloat { - signed: false, - p, - rd, - rn, - }); - } - 0x5ea1_b800 => { - return Some(Op::ScalarFloatToInt { - signed: true, - p, - rd, - rn, - }); - } - 0x7ea1_b800 => { - return Some(Op::ScalarFloatToInt { - signed: false, - p, - rd, - rn, - }); - } - _ => {} - } - // Vector `fabs` (U=0) and `fneg` (U=1). - if word & 0x9fbf_fc00 == 0x0ea0_f800 { - let (q, double) = (word & 0x4000_0000 != 0, word & 0x0040_0000 != 0); - // `.1d` is reserved. - if double && !q { - return None; - } - return Some(Op::VectorSign { - negate: word & 0x2000_0000 != 0, - double, - q, - rd, - rn, - }); - } - None -} - -/// Whether [`run`] implements `word`. -pub fn is_supported(word: u32) -> bool { - parse(word).is_some() -} +use volt_isa_aarch64::decode::expand_immediate; +use volt_isa_aarch64::fp::{Binary, Fused, Op, Precision, Unary}; trait Float: Copy + PartialOrd { const SIGN: u64; @@ -538,10 +214,8 @@ macro_rules! at { }; } -/// Run the floating-point instruction `word` on `cpu`. `word` must satisfy -/// [`is_supported`]. -pub fn run(cpu: &mut Cpu, word: u32) { - let op = parse(word).expect("an unsupported floating-point word reached the executor"); +/// Run the decoded floating-point instruction `op` on `cpu`. +pub fn run(cpu: &mut Cpu, op: Op) { match op { Op::Unary { kind, p, rd, rn } => { let a = read(cpu, p, rn); @@ -602,7 +276,8 @@ pub fn run(cpu: &mut Cpu, word: u32) { at!(p, T => fused(kind, T::from_u64(n), T::from_u64(m), T::from_u64(a)).to_u64()); write(cpu, rd, r); } - Op::Compare { p, rn, rm } => { + // FPSR exception flags are not modelled, so `fcmpe` executes as `fcmp`. + Op::Compare { p, rn, rm, .. } => { let a = read(cpu, p, rn); let b = rm.map_or(0, |rm| read(cpu, p, rm)); let nzcv = at!(p, T => compare(T::from_u64(a), T::from_u64(b))); @@ -728,6 +403,11 @@ fn float_to_int(bits: u64, signed: bool, wide: bool, p: Precision) -> u64 { mod tests { use super::*; + /// Decode `word` and run it, as the machine does for a trapped host word. + fn run(cpu: &mut Cpu, word: u32) { + super::run(cpu, volt_isa_aarch64::fp::decode(word).expect("an fp word")); + } + fn cpu() -> Cpu { Cpu::default() } @@ -966,19 +646,6 @@ mod tests { assert_eq!(c.v[0], [0x3f80_0000_4000_0000, 0]); } - #[test] - fn unsupported_encodings_are_not_claimed() { - // fmov d0, x1 and fmov x0, d1 are `SimdFmov`, not conversions. - assert!(!is_supported(0x9e670020)); - assert!(!is_supported(0x9e660020)); - // Half precision (type 3) and reserved type 2. - assert!(!is_supported(0x1ee22820)); - assert!(!is_supported(0x1ea22820)); - // fneg v0.1d is reserved. - assert!(!is_supported(0x2ee0f820)); - assert!(is_supported(0x1e620820)); - } - #[test] fn the_immediate_expansion_matches_the_architectural_pattern() { assert_eq!(expand_immediate(0x70, false), 0x3f80_0000); diff --git a/src/emulator/src/aarch64/gxf.rs b/src/emulator/src/aarch64/gxf.rs index 7b3f6c4..9fd9f9f 100644 --- a/src/emulator/src/aarch64/gxf.rs +++ b/src/emulator/src/aarch64/gxf.rs @@ -1,6 +1,6 @@ //! Guarded execution (GXF): `genter` and `gexit`, the instructions that move between //! EL and the guarded levels GL1 and GL2 (Asahi Linux documentation, "Apple Proprietary -//! Instructions" and "SPRR and GXF"). +//! Instructions" and "SPRR and GXF"). Decoding lives in `volt_isa_aarch64::gxf`. //! //! The feature is off by default and only a monitor firmware (SPTM or TXM) enables it, //! which this model does not run. Its enable bit reads as zero, so both instructions @@ -8,22 +8,12 @@ //! default" without naming the exception class; undefined-instruction is an assumption. use super::{Cpu, exception}; - -/// `gexit`: leave guarded mode. -const GEXIT: u32 = 0x0020_1400; -/// `genter` with its `imm5` in bits 4:0 (stored in `ESR_GLx[5:0]`). -const GENTER: u32 = 0x0020_1420; - -/// Whether `word` is `genter` or `gexit`. -pub fn is_supported(word: u32) -> bool { - word == GEXIT || word & !0x1f == GENTER -} +use volt_isa_aarch64::gxf::Op; /// Take the undefined-instruction exception `genter` or `gexit` raises while GXF is off. /// `cpu.pc` is the address of the next instruction; the exception returns to the /// instruction itself. -pub fn run(cpu: &mut Cpu, word: u32) { - debug_assert!(is_supported(word)); +pub fn run(cpu: &mut Cpu, _op: Op) { cpu.pc = cpu.pc.wrapping_sub(4); let pc = cpu.pc; exception::take( @@ -42,15 +32,6 @@ pub fn run(cpu: &mut Cpu, word: u32) { mod tests { use super::*; - #[test] - fn genter_and_gexit_are_recognised_with_any_genter_immediate() { - assert!(is_supported(GEXIT)); - assert!(is_supported(GENTER)); - assert!(is_supported(GENTER | 0x1f)); - assert!(!is_supported(0xd503_201f)); - assert!(!is_supported(GEXIT + 4)); - } - #[test] fn with_guarded_execution_off_genter_is_an_undefined_instruction_at_the_instruction() { let mut cpu = Cpu::default(); @@ -59,7 +40,7 @@ mod tests { cpu.system.vbar_el1 = 0x4000_1000; // The executor runs after the instruction at 0x4000_0000 has advanced `pc`. cpu.pc = 0x4000_0004; - run(&mut cpu, GENTER); + run(&mut cpu, Op::Genter { imm5: 0 }); assert_eq!( cpu.system.elr_el1, 0x4000_0000, "returns to the genter itself" diff --git a/src/emulator/src/aarch64/host.rs b/src/emulator/src/aarch64/host.rs index 9cc6da1..bfcf0fb 100644 --- a/src/emulator/src/aarch64/host.rs +++ b/src/emulator/src/aarch64/host.rs @@ -1,62 +1,110 @@ //! Instructions the JIT does not compile. It stores the raw word in `Cpu::value` //! and traps; the machine (or user mode) then runs the word on the host here. //! Floating point and the wider integer SIMD families live in [`fp`] and [`simd`]. +//! +//! Which words these are is decided by the one decoder, `volt_isa_aarch64::decode`: +//! the instruction variants [`runs_on_host`] names. use super::{Cpu, fp, gxf, pauth, simd}; -use volt_target::aarch64::decode::{DecodeError, Instruction, decode}; - -/// Whether [`run`] implements `word`. -pub fn is_supported(word: u32) -> bool { - fp::is_supported(word) - || simd::is_supported(word) - || pauth::is_supported(word) - || gxf::is_supported(word) -} +use volt_isa_aarch64::decode::{Instruction, decode}; -/// What the emulator does with one instruction word: the host runs the words [`run`] -/// implements, and the JIT compiles every other instruction the decoder recognises. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Dispatch { - /// A structured instruction for the JIT. - Guest(Instruction), - /// A word for [`run`]. It ends a block, the same as a trap does. - Host(u32), +/// Whether the emulator runs `instruction` on the host instead of lifting it. +pub fn runs_on_host(instruction: &Instruction) -> bool { + matches!( + instruction, + Instruction::Fp(_) | Instruction::Simd(_) | Instruction::Pauth(_) | Instruction::Gxf(_) + ) } -impl Dispatch { - /// Whether this word ends a translated block; see `Instruction::terminates_with`. - pub fn terminates_with(self, inline_memory: bool) -> bool { - match self { - Self::Guest(instruction) => instruction.terminates_with(inline_memory), - Self::Host(_) => true, - } +/// Whether the emulator ends a translated block after `instruction`. +/// +/// This is policy of this emulator's [`CpuEnvironment`](super::environment), not a +/// property of the instruction set: everything the environment forwards to the +/// machine (memory, exclusives, atomics, system events, words run on the host) must +/// leave the block so the machine can run it, and so must every architectural +/// branch. With `inline_memory`, plain loads and stores are compiled with an inline +/// data-TLB fast path whose slow path exits mid-block, so they no longer end it. +pub fn ends_block(instruction: &Instruction, inline_memory: bool) -> bool { + use Instruction::*; + if runs_on_host(instruction) { + return true; } - - /// [`Self::terminates_with`] without inline memory. - pub fn terminates(self) -> bool { - self.terminates_with(false) + if inline_memory + && matches!( + instruction, + Memory(_) | Pair(_) | Literal(_) | SimdMemory(_) | SimdPair(_) + ) + { + return false; } + matches!( + instruction, + Memory(_) + | Exclusive(_) + | Atomic(_) + | Literal(_) + | SimdLiteral(_) + | Pair(_) + | SimdPair(_) + | SimdMemory(_) + | SimdStruct(_) + | B(_) + | BCond(_) + | TestBranch(_) + | Call(_) + | Indirect(_) + | Svc(_) + | Psci + | Wfi + | Wfe + | Brk(_) + | Isb(_) + | DcZva(_) + | AddressTranslate(_) + | Eret + | Tlbi(_) + ) } -/// Classify `word` for execution. A word [`is_supported`] names is never compiled, -/// even where the decoder has a structured form for it. -pub fn dispatch(word: u32) -> Result { - if is_supported(word) { - Ok(Dispatch::Host(word)) - } else { - decode(word).map(Dispatch::Guest) +/// Run `word` on `cpu`. `word` must decode to an instruction [`runs_on_host`] names. +pub fn run(cpu: &mut Cpu, word: u32) { + match decode(word) { + Ok(Instruction::Fp(op)) => fp::run(cpu, op), + Ok(Instruction::Simd(op)) => simd::run(cpu, op), + Ok(Instruction::Pauth(op)) => pauth::run(cpu, op), + Ok(Instruction::Gxf(op)) => gxf::run(cpu, op), + other => unreachable!("word {word:08x} is not a host instruction: {other:?}"), } } -/// Run `word` on `cpu`. `word` must satisfy [`is_supported`]. -pub fn run(cpu: &mut Cpu, word: u32) { - if fp::is_supported(word) { - fp::run(cpu, word) - } else if simd::is_supported(word) { - simd::run(cpu, word) - } else if gxf::is_supported(word) { - gxf::run(cpu, word) - } else { - pauth::run(cpu, word) +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_policy_cuts_at_machine_effects_and_branches() { + let word = |w| decode(w).unwrap(); + assert!(!ends_block(&word(0xd503309f), false)); // dsb + assert!(ends_block(&word(0xd5033fdf), false)); // isb + assert!(ends_block(&word(0xd69f03e0), false)); // eret + assert!(ends_block(&word(0x1400_0000), false)); // b . + assert!(ends_block(&word(0x1e22_2820), true)); // fadd s0, s1, s2: runs on the host + } + + #[test] + fn inline_memory_only_lets_plain_accesses_continue() { + let load = decode(0xf940_0020).unwrap(); // ldr x0, [x1] + let exclusive = decode(0xc85f_7c20).unwrap(); // ldxr x0, [x1] + assert!(ends_block(&load, false)); + assert!(!ends_block(&load, true)); + assert!(ends_block(&exclusive, true)); + } + + #[test] + fn floating_point_pointer_authentication_and_integer_simd_run_on_the_host() { + assert!(runs_on_host(&decode(0x1e22_2820).unwrap())); // fadd s0, s1, s2 + assert!(runs_on_host(&decode(0xdac1_0020).unwrap())); // pacia x0, x1 + assert!(runs_on_host(&decode(0x4e82_1820).unwrap())); // uzp1 v0.4s, v1.4s, v2.4s + assert!(!runs_on_host(&decode(0x8b02_0020).unwrap())); // add x0, x1, x2: lifted } } diff --git a/src/emulator/src/aarch64/identification.rs b/src/emulator/src/aarch64/identification.rs index 87fd5c5..67a47d0 100644 --- a/src/emulator/src/aarch64/identification.rs +++ b/src/emulator/src/aarch64/identification.rs @@ -1,4 +1,4 @@ -use volt_target::aarch64::decode::SystemRegister; +use volt_isa_aarch64::decode::SystemRegister; #[allow(non_upper_case_globals)] pub const counter_hz: u64 = 62_500_000; /// Cache hierarchy `CLIDR_EL1` reports: split L1, unified L2, 64-byte lines. diff --git a/src/emulator/src/aarch64/pauth.rs b/src/emulator/src/aarch64/pauth.rs index 492dc10..3fed138 100644 --- a/src/emulator/src/aarch64/pauth.rs +++ b/src/emulator/src/aarch64/pauth.rs @@ -1,6 +1,7 @@ //! Pointer authentication (FEAT_PAuth): the QARMA5 `ComputePAC` the architecture //! defines, the sign, authenticate and strip operations on 48-bit pointers, and the -//! PAuth instructions the JIT hands to the host ([`super::host`]). +//! PAuth instructions the JIT hands to the host ([`super::host`]); decoding lives in +//! `volt_isa_aarch64::pauth`. //! //! The emulated CPU advertises the architected QARMA5 algorithm without TBI, EPAC or //! FPAC (`ID_AA64ISAR1_EL1.APA` = 1). A pointer's PAC then occupies bits 63:56 and 54:48, @@ -10,6 +11,7 @@ //! `target/arm/tcg/pauth_helper.c`; the tests check it against that implementation. use super::{Cpu, exception}; +use volt_isa_aarch64::pauth::{KeyId, Modifier, Op}; /// A 128-bit PAuth key: `hi` is bits 127:64 (`APxAKeyHi_EL1`), `lo` bits 63:0. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -18,289 +20,6 @@ pub struct Key { pub lo: u64, } -/// The key the instruction uses, as an index into [`super::cpu::System::pac_keys`] -/// (two words per key, low word first). -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum KeyId { - Ia, - Ib, - Da, - Db, - Ga, -} - -impl KeyId { - fn index(self) -> usize { - match self { - Self::Ia => 0, - Self::Ib => 1, - Self::Da => 2, - Self::Db => 3, - Self::Ga => 4, - } - } - - /// The `SCTLR_EL1` bit that enables the key, if any. The generic key has none. - fn enable_bit(self) -> Option { - match self { - Self::Ia => Some(31), - Self::Ib => Some(30), - Self::Da => Some(27), - Self::Db => Some(13), - Self::Ga => None, - } - } - - /// The `keynumber` the architecture reports in an authentication failure. - fn number(self) -> u64 { - match self { - Self::Ia => 0, - Self::Ib => 1, - Self::Da => 2, - Self::Db => 3, - Self::Ga => 4, - } - } -} - -/// The modifier operand: zero, or a register where register 31 is `SP`. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Modifier { - Zero, - Reg(u8), -} - -/// One PAuth instruction, decoded. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Op { - Sign { - key: KeyId, - reg: u8, - modifier: Modifier, - }, - Auth { - key: KeyId, - reg: u8, - modifier: Modifier, - }, - Strip { - reg: u8, - }, - Pacga { - rd: u8, - rn: u8, - modifier: Modifier, - }, - /// `BRAA`, `BRAB`, `BLRAA`, `BLRAB` and their zero-modifier forms. - Branch { - key: KeyId, - link: bool, - target: u8, - modifier: Modifier, - }, - /// `RETAA` and `RETAB`: return to `LR`, authenticated with `SP`. - Return { - key: KeyId, - }, - /// `ERETAA` and `ERETAB`: return from the exception, authenticating `ELR_EL1`. - ExceptionReturn { - key: KeyId, - }, -} - -/// The hint-space forms (`PACIASP` and friends), which are no-ops without PAuth. -const HINTS: [(u32, Op); 13] = [ - ( - 0xd503_233f, - Op::Sign { - key: KeyId::Ia, - reg: 30, - modifier: Modifier::Reg(31), - }, - ), - ( - 0xd503_23bf, - Op::Auth { - key: KeyId::Ia, - reg: 30, - modifier: Modifier::Reg(31), - }, - ), - ( - 0xd503_237f, - Op::Sign { - key: KeyId::Ib, - reg: 30, - modifier: Modifier::Reg(31), - }, - ), - ( - 0xd503_23ff, - Op::Auth { - key: KeyId::Ib, - reg: 30, - modifier: Modifier::Reg(31), - }, - ), - ( - 0xd503_211f, - Op::Sign { - key: KeyId::Ia, - reg: 17, - modifier: Modifier::Reg(16), - }, - ), - ( - 0xd503_219f, - Op::Auth { - key: KeyId::Ia, - reg: 17, - modifier: Modifier::Reg(16), - }, - ), - ( - 0xd503_215f, - Op::Sign { - key: KeyId::Ib, - reg: 17, - modifier: Modifier::Reg(16), - }, - ), - ( - 0xd503_21df, - Op::Auth { - key: KeyId::Ib, - reg: 17, - modifier: Modifier::Reg(16), - }, - ), - ( - 0xd503_231f, - Op::Sign { - key: KeyId::Ia, - reg: 30, - modifier: Modifier::Zero, - }, - ), - ( - 0xd503_239f, - Op::Auth { - key: KeyId::Ia, - reg: 30, - modifier: Modifier::Zero, - }, - ), - ( - 0xd503_235f, - Op::Sign { - key: KeyId::Ib, - reg: 30, - modifier: Modifier::Zero, - }, - ), - ( - 0xd503_23df, - Op::Auth { - key: KeyId::Ib, - reg: 30, - modifier: Modifier::Zero, - }, - ), - (0xd503_20ff, Op::Strip { reg: 30 }), -]; - -fn decode_word(word: u32) -> Option { - let rd = (word & 31) as u8; - let rn = ((word >> 5) & 31) as u8; - let rm = rd; - // Data-processing, one source: PAC and AUT (opcodes 0-15), and XPAC (16-17). - if word & 0xffff_0000 == 0xdac1_0000 { - let opcode = (word >> 10) & 0x3f; - let key = [KeyId::Ia, KeyId::Ib, KeyId::Da, KeyId::Db][(opcode & 3) as usize]; - let modifier = if opcode & 8 != 0 { - Modifier::Zero - } else { - Modifier::Reg(rn) - }; - return match opcode { - 0..=15 if opcode & 4 == 0 => Some(Op::Sign { - key, - reg: rd, - modifier, - }), - 0..=15 => Some(Op::Auth { - key, - reg: rd, - modifier, - }), - 16 | 17 if rn == 31 => Some(Op::Strip { reg: rd }), - _ => None, - }; - } - // Data-processing, two sources: PACGA Xd, Xn, Xm|SP. - if word & 0xffe0_fc00 == 0x9ac0_3000 { - let rm = ((word >> 16) & 31) as u8; - return Some(Op::Pacga { - rd, - rn, - modifier: Modifier::Reg(rm), - }); - } - // Branches with authentication. Bit 10 selects the B key; bit 11 is always set. - let b_key = if word & 0x400 != 0 { - KeyId::Ib - } else { - KeyId::Ia - }; - if word & 0xffff_f800 == 0xd71f_0800 { - return Some(Op::Branch { - key: b_key, - link: false, - target: rn, - modifier: Modifier::Reg(rm), - }); - } - if word & 0xffff_f81f == 0xd61f_081f { - return Some(Op::Branch { - key: b_key, - link: false, - target: rn, - modifier: Modifier::Zero, - }); - } - if word & 0xffff_f800 == 0xd73f_0800 { - return Some(Op::Branch { - key: b_key, - link: true, - target: rn, - modifier: Modifier::Reg(rm), - }); - } - if word & 0xffff_f81f == 0xd63f_081f { - return Some(Op::Branch { - key: b_key, - link: true, - target: rn, - modifier: Modifier::Zero, - }); - } - if word & 0xffff_fbff == 0xd65f_0bff { - return Some(Op::Return { key: b_key }); - } - if word & 0xffff_fbff == 0xd69f_0bff { - return Some(Op::ExceptionReturn { key: b_key }); - } - HINTS - .iter() - .find(|(hint, _)| *hint == word) - .map(|&(_, op)| op) -} - -/// Whether [`run`] implements `word`. -pub fn is_supported(word: u32) -> bool { - decode_word(word).is_some() -} - /// The 48-bit pointer's bits 63:48 all equal bit 47, so the pointer is canonical. fn is_canonical(ptr: u64) -> bool { let top = ptr >> 48; @@ -556,11 +275,22 @@ pub fn compute_pac(data: u64, modifier: u64, key: Key) -> u64 { working ^ modk0 } +/// Index of `id` into `System::pac_keys` (two words per key, low word first). +fn key_slot(id: KeyId) -> usize { + match id { + KeyId::Ia => 0, + KeyId::Ib => 1, + KeyId::Da => 2, + KeyId::Db => 3, + KeyId::Ga => 4, + } +} + fn key(cpu: &Cpu, id: KeyId) -> Key { let keys = &cpu.system.pac_keys; Key { - lo: keys[2 * id.index()], - hi: keys[2 * id.index() + 1], + lo: keys[2 * key_slot(id)], + hi: keys[2 * key_slot(id) + 1], } } @@ -593,15 +323,16 @@ fn modifier_value(cpu: &Cpu, modifier: Modifier) -> u64 { } } -/// Execute the PAuth instruction `word` on `cpu`. `word` must satisfy [`is_supported`]. +/// Execute the decoded PAuth instruction `op` on `cpu`. /// `cpu.pc` is the address of the next instruction, which a link writes to `LR`. -pub fn run(cpu: &mut Cpu, word: u32) { - let op = decode_word(word).expect("pauth::run is only called for supported words"); +pub fn run(cpu: &mut Cpu, op: Op) { match op { + // `hint` only selects the disassembly spelling (`paciasp` vs `pacia x30, sp`). Op::Sign { key: id, reg, modifier, + hint: _, } => { if enabled(cpu, id) { let value = add_pac( @@ -616,6 +347,7 @@ pub fn run(cpu: &mut Cpu, word: u32) { key: id, reg, modifier, + hint: _, } => { if enabled(cpu, id) { let value = auth( @@ -627,7 +359,8 @@ pub fn run(cpu: &mut Cpu, word: u32) { write_x(cpu, reg, value); } } - Op::Strip { reg } => { + // `xpaci` and `xpacd` strip identically here. + Op::Strip { reg, .. } => { let value = strip(cpu.x[reg as usize]); write_x(cpu, reg, value); } @@ -688,6 +421,10 @@ mod tests { lo: 0x84be_85ce_9804_e94b, }; + fn op(word: u32) -> Op { + volt_isa_aarch64::pauth::decode(word).unwrap() + } + /// `ComputePAC` values from QEMU's `pauth_computepac_architected`, compiled from its /// source with the same key: `(data, modifier, pac)`. const REFERENCE: [(u64, u64, u64); 6] = [ @@ -782,69 +519,6 @@ mod tests { ); } - #[test] - fn the_instruction_encodings_decode_as_their_names() { - // Assembled with GNU as for armv8.3-a. - assert_eq!( - decode_word(0xdac1_0020), - Some(Op::Sign { - key: KeyId::Ia, - reg: 0, - modifier: Modifier::Reg(1) - }) - ); - assert_eq!( - decode_word(0xdac1_11ac), - Some(Op::Auth { - key: KeyId::Ia, - reg: 12, - modifier: Modifier::Reg(13) - }) - ); - assert_eq!( - decode_word(0xdac1_23f0), - Some(Op::Sign { - key: KeyId::Ia, - reg: 16, - modifier: Modifier::Zero - }) - ); - assert_eq!(decode_word(0xdac1_43f8), Some(Op::Strip { reg: 24 })); - assert_eq!( - decode_word(0xd71f_0c43), - Some(Op::Branch { - key: KeyId::Ib, - link: false, - target: 2, - modifier: Modifier::Reg(3) - }) - ); - assert_eq!( - decode_word(0xd63f_095f), - Some(Op::Branch { - key: KeyId::Ia, - link: true, - target: 10, - modifier: Modifier::Zero - }) - ); - assert_eq!( - decode_word(0xd65f_0fff), - Some(Op::Return { key: KeyId::Ib }) - ); - assert_eq!( - decode_word(0xd503_233f), - Some(Op::Sign { - key: KeyId::Ia, - reg: 30, - modifier: Modifier::Reg(31) - }) - ); - // Plain BR and ordinary one-source data processing are not PAuth. - assert!(!is_supported(0xd61f_0000)); - assert!(!is_supported(0xdac0_0000)); - } - #[test] fn a_pac_instruction_signs_its_register_and_a_branch_jumps_to_the_authenticated_target() { let mut cpu = Cpu::default(); @@ -854,12 +528,12 @@ mod tests { let ptr = 0x0000_0008_1234_5678; cpu.x[0] = ptr; cpu.x[1] = 0x2222; - run(&mut cpu, 0xdac1_0020); + run(&mut cpu, op(0xdac1_0020)); assert_eq!(cpu.x[0], add_pac(ptr, 0x2222, KEY)); cpu.x[2] = add_pac(0x4000, 0x2222, KEY); cpu.x[3] = 0x2222; cpu.pc = 0x1000; - run(&mut cpu, 0xd71f_0800 | 2 << 5 | 3); + run(&mut cpu, op(0xd71f_0800 | 2 << 5 | 3)); assert_eq!(cpu.pc, 0x4000); } } diff --git a/src/emulator/src/aarch64/simd.rs b/src/emulator/src/aarch64/simd.rs index 3cced27..cc8bf10 100644 --- a/src/emulator/src/aarch64/simd.rs +++ b/src/emulator/src/aarch64/simd.rs @@ -5,432 +5,11 @@ //! wide arithmetic (`uaddl`, `umull`, ...), and the 64-bit scalar forms. //! //! Each register is a `u128`, each lane a bit field of it, so one lane loop serves -//! every element size. Decode only classifies a word with [`is_supported`]; the -//! JIT traps and the machine (or user mode) calls [`run`]. +//! every element size. Decoding lives in `volt_isa_aarch64::simd`; the JIT traps and +//! the machine (or user mode) calls [`run`]. use super::Cpu; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Permute { - Uzp1, - Uzp2, - Zip1, - Zip2, - Trn1, - Trn2, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Zero { - Gt, - Ge, - Le, - Lt, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Narrow { - Plain, - SignedSat, - UnsignedSat, - SignedToUnsignedSat, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Long { - Add, - AddWide, - Sub, - SubWide, - Abd, - Mul, - Mla, - Mls, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Scalar { - Add, - Sub, - Gt, - Hi, - Ge, - Hs, - Eq, - Tst, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Op { - Permute { - kind: Permute, - size: u32, - q: bool, - rd: u8, - rn: u8, - rm: u8, - }, - /// `rev16`/`rev32`/`rev64`: reverse the lanes inside each group of `group` bits. - Reverse { - group: u32, - size: u32, - q: bool, - rd: u8, - rn: u8, - }, - Count { - q: bool, - rd: u8, - rn: u8, - }, - Not { - q: bool, - rd: u8, - rn: u8, - }, - ReverseBits { - q: bool, - rd: u8, - rn: u8, - }, - CompareZero { - kind: Zero, - size: u32, - q: bool, - rd: u8, - rn: u8, - }, - Abs { - negate: bool, - size: u32, - q: bool, - rd: u8, - rn: u8, - }, - /// `xtn`/`sqxtn`/`uqxtn`/`sqxtun`; `upper` is the `2` form (keeps the low half). - Narrow { - kind: Narrow, - size: u32, - upper: bool, - rd: u8, - rn: u8, - }, - /// `saddlp`/`uaddlp` and, with `accumulate`, `sadalp`/`uadalp`. - PairLong { - signed: bool, - accumulate: bool, - size: u32, - q: bool, - rd: u8, - rn: u8, - }, - Long { - kind: Long, - signed: bool, - size: u32, - upper: bool, - rd: u8, - rn: u8, - rm: u8, - }, - ScalarAddp { - rd: u8, - rn: u8, - }, - ScalarDup { - size: u32, - index: u32, - rd: u8, - rn: u8, - }, - ScalarThree { - kind: Scalar, - rd: u8, - rn: u8, - rm: u8, - }, - /// `shl`, and `sshr`/`ushr`, by an immediate. - ShiftImm { - right: bool, - signed: bool, - size: u32, - q: bool, - shift: u32, - rd: u8, - rn: u8, - }, - /// `sshll`/`ushll` (`sxtl`/`uxtl` at shift 0); `upper` is the `2` form. - ShiftLong { - signed: bool, - size: u32, - shift: u32, - upper: bool, - rd: u8, - rn: u8, - }, - /// `shrn`/`shrn2`. - ShiftNarrow { - size: u32, - shift: u32, - upper: bool, - rd: u8, - rn: u8, - }, -} - -fn parse(word: u32) -> Option { - let rd = (word & 31) as u8; - let rn = ((word >> 5) & 31) as u8; - let rm = ((word >> 16) & 31) as u8; - let q = word & 0x4000_0000 != 0; - let u = word & 0x2000_0000 != 0; - let size = (word >> 22) & 3; - // Permute: bit 21 clear, bits 11:10 = 10, bits 14:12 pick the operation. - if word & 0xbf20_8c00 == 0x0e00_0800 { - let kind = match (word >> 12) & 7 { - 1 => Permute::Uzp1, - 2 => Permute::Trn1, - 3 => Permute::Zip1, - 5 => Permute::Uzp2, - 6 => Permute::Trn2, - 7 => Permute::Zip2, - _ => return None, - }; - // 64-bit lanes need the full vector. - if size == 3 && !q { - return None; - } - return Some(Op::Permute { - kind, - size, - q, - rd, - rn, - rm, - }); - } - // Two-register miscellaneous: bits 21:17 = 10000, bits 11:10 = 10. - if word & 0x9f3e_0c00 == 0x0e20_0800 { - let opcode = (word >> 12) & 0x1f; - let wide_needs_q = size != 3 || q; - return match (u, opcode) { - (false, 0) if size < 3 => Some(Op::Reverse { - group: 64, - size, - q, - rd, - rn, - }), - (true, 0) if size < 2 => Some(Op::Reverse { - group: 32, - size, - q, - rd, - rn, - }), - (false, 1) if size == 0 => Some(Op::Reverse { - group: 16, - size, - q, - rd, - rn, - }), - (false | true, 2) if size < 3 => Some(Op::PairLong { - signed: !u, - accumulate: false, - size, - q, - rd, - rn, - }), - (false | true, 6) if size < 3 => Some(Op::PairLong { - signed: !u, - accumulate: true, - size, - q, - rd, - rn, - }), - (false, 5) if size == 0 => Some(Op::Count { q, rd, rn }), - (true, 5) if size == 0 => Some(Op::Not { q, rd, rn }), - (true, 5) if size == 1 => Some(Op::ReverseBits { q, rd, rn }), - (false, 8) if wide_needs_q => Some(Op::CompareZero { - kind: Zero::Gt, - size, - q, - rd, - rn, - }), - (true, 8) if wide_needs_q => Some(Op::CompareZero { - kind: Zero::Ge, - size, - q, - rd, - rn, - }), - (true, 9) if wide_needs_q => Some(Op::CompareZero { - kind: Zero::Le, - size, - q, - rd, - rn, - }), - (false, 10) if wide_needs_q => Some(Op::CompareZero { - kind: Zero::Lt, - size, - q, - rd, - rn, - }), - (false, 11) if wide_needs_q => Some(Op::Abs { - negate: false, - size, - q, - rd, - rn, - }), - (true, 11) if wide_needs_q => Some(Op::Abs { - negate: true, - size, - q, - rd, - rn, - }), - (false, 0x12) if size < 3 => Some(Op::Narrow { - kind: Narrow::Plain, - size, - upper: q, - rd, - rn, - }), - (true, 0x12) if size < 3 => Some(Op::Narrow { - kind: Narrow::SignedToUnsignedSat, - size, - upper: q, - rd, - rn, - }), - (false, 0x14) if size < 3 => Some(Op::Narrow { - kind: Narrow::SignedSat, - size, - upper: q, - rd, - rn, - }), - (true, 0x14) if size < 3 => Some(Op::Narrow { - kind: Narrow::UnsignedSat, - size, - upper: q, - rd, - rn, - }), - // `cmeq #0` (U=0, opcode 9) is `SimdCompareZero`. - _ => None, - }; - } - // Three registers different: bit 21 set, bits 11:10 = 00. - if word & 0x9f20_0c00 == 0x0e20_0000 && size < 3 { - let kind = match (word >> 12) & 15 { - 0 => Long::Add, - 1 => Long::AddWide, - 2 => Long::Sub, - 3 => Long::SubWide, - 7 => Long::Abd, - 8 => Long::Mla, - 10 => Long::Mls, - 12 => Long::Mul, - _ => return None, - }; - return Some(Op::Long { - kind, - signed: !u, - size, - upper: q, - rd, - rn, - rm, - }); - } - // Shift by immediate: `immh` (bits 22:19) gives the element size and, with - // `immb`, the shift. `immh == 0` is the modified-immediate class instead. - if word & 0x9f80_0400 == 0x0f00_0400 && (word >> 19) & 15 != 0 { - let immh = (word >> 19) & 15; - let immhb = (word >> 16) & 0x7f; - // The position of the top set bit of immh: 0 for bytes up to 3 for doublewords. - let size = 31 - immh.leading_zeros(); - let esize = 8 << size; - return match (u, (word >> 11) & 0x1f) { - (false, 0b01010) if size < 3 || q => Some(Op::ShiftImm { - right: false, - signed: false, - size, - q, - shift: immhb - esize, - rd, - rn, - }), - (_, 0b00000) if size < 3 || q => Some(Op::ShiftImm { - right: true, - signed: !u, - size, - q, - shift: 2 * esize - immhb, - rd, - rn, - }), - (_, 0b10100) if size < 3 => Some(Op::ShiftLong { - signed: !u, - size, - shift: immhb - esize, - upper: q, - rd, - rn, - }), - (false, 0b10000) if size < 3 => Some(Op::ShiftNarrow { - size, - shift: 2 * esize - immhb, - upper: q, - rd, - rn, - }), - _ => None, - }; - } - // Scalar pairwise `addp Dd, Vn.2D`. - if word & 0xffff_fc00 == 0x5ef1_b800 { - return Some(Op::ScalarAddp { rd, rn }); - } - // Scalar `dup`: `mov Bd/Hd/Sd/Dd, Vn.T[index]`. - if word & 0xffe0_fc00 == 0x5e00_0400 { - let imm5 = (word >> 16) & 0x1f; - if imm5 == 0 || imm5.trailing_zeros() > 3 { - return None; - } - let size = imm5.trailing_zeros(); - return Some(Op::ScalarDup { - size, - index: imm5 >> (size + 1), - rd, - rn, - }); - } - // Scalar three-same, 64-bit only. - if word & 0xdf20_0400 == 0x5e20_0400 && size == 3 { - let kind = match (u, (word >> 11) & 0x1f) { - (false, 0x10) => Scalar::Add, - (true, 0x10) => Scalar::Sub, - (false, 0x06) => Scalar::Gt, - (true, 0x06) => Scalar::Hi, - (false, 0x07) => Scalar::Ge, - (true, 0x07) => Scalar::Hs, - (true, 0x11) => Scalar::Eq, - (false, 0x11) => Scalar::Tst, - _ => return None, - }; - return Some(Op::ScalarThree { kind, rd, rn, rm }); - } - None -} - -/// Whether [`run`] implements `word`. -pub fn is_supported(word: u32) -> bool { - parse(word).is_some() -} +use volt_isa_aarch64::simd::{Long, Narrow, Op, Permute, Scalar, Zero}; fn read(cpu: &Cpu, reg: u8) -> u128 { let [lo, hi] = cpu.v[usize::from(reg)]; @@ -470,9 +49,8 @@ fn lanes(size: u32, q: bool) -> u32 { (if q { 128 } else { 64 }) / (8 << size) } -/// Run the instruction `word` on `cpu`. `word` must satisfy [`is_supported`]. -pub fn run(cpu: &mut Cpu, word: u32) { - let op = parse(word).expect("an unsupported SIMD word reached the executor"); +/// Run the decoded instruction `op` on `cpu`. +pub fn run(cpu: &mut Cpu, op: Op) { match op { Op::Permute { kind, @@ -789,6 +367,14 @@ pub fn run(cpu: &mut Cpu, word: u32) { mod tests { use super::*; + /// Decode `word` and run it, as the machine does for a trapped host word. + fn run(cpu: &mut Cpu, word: u32) { + super::run( + cpu, + volt_isa_aarch64::simd::decode(word).expect("a SIMD word"), + ); + } + /// A register whose lane `i` of `bits` bits is `f(i)`. fn pack(bits: u32, count: u32, f: impl Fn(u32) -> u128) -> [u64; 2] { let mut v = 0u128; @@ -1150,14 +736,4 @@ mod tests { run(&mut c, 0x0f3f8420); assert_eq!(c.v[0], [0x0000_0005_8000_0003, 0]); } - #[test] - fn classification_declines_what_it_does_not_implement() { - // cmeq #0 is `SimdCompareZero`; half-width lanes need Q for 64-bit lanes; - // `ins`, `umov` and the three-same group belong to other decoders. - assert!(!is_supported(0x0e209820)); - assert!(!is_supported(0x0ee0b820), "abs .1d is reserved"); - assert!(!is_supported(0x2ee21c20), "bif is three-same"); - assert!(!is_supported(0x4e083c20), "umov"); - assert!(is_supported(0x4e821820)); - } } diff --git a/src/emulator/src/aarch64/simd_struct.rs b/src/emulator/src/aarch64/simd_struct.rs index 81f7dc0..59b7b5b 100644 --- a/src/emulator/src/aarch64/simd_struct.rs +++ b/src/emulator/src/aarch64/simd_struct.rs @@ -6,7 +6,7 @@ //! [`execute`] against its own memory, one element at a time, so each backend keeps //! its own translation, device and fault handling. -use volt_target::aarch64::simd_struct::{Shape, StructDesc}; +use volt_isa_aarch64::simd_struct::{Shape, StructDesc}; fn mask(esize: u8) -> u64 { if esize >= 8 { diff --git a/src/emulator/src/system/m1.rs b/src/emulator/src/system/m1.rs index af0af59..1cc7af9 100644 --- a/src/emulator/src/system/m1.rs +++ b/src/emulator/src/system/m1.rs @@ -60,7 +60,7 @@ pub fn identify(cpu: &mut crate::aarch64::cpu::Cpu) { // The SPTM monitor polls `S3_6_c15_c12_4` until bit 0 is set before it continues // (`mrs x0, s3_6_c15_c12_4; and x1, x0, #1; cbz x1, `). Nothing here names the // register, so bit 0 reads as set, which is the only value that lets the loop exit. - cpu.apple_bank.0[volt_target::aarch64::decode::apple_bank_slot(6, 12, 4)] = 1; + cpu.apple_bank.0[volt_isa_aarch64::decode::apple_bank_slot(6, 12, 4)] = 1; } /// The SoC devices on the bus: the AIC, the power-state blocks, the console, and the diff --git a/src/emulator/src/system/machine.rs b/src/emulator/src/system/machine.rs index 336d4ec..146ea6b 100644 --- a/src/emulator/src/system/machine.rs +++ b/src/emulator/src/system/machine.rs @@ -7,7 +7,7 @@ use crate::aarch64::{Cache, Cpu}; use crate::aarch64::{exception, translate}; use crate::devices::{bus::Device, gicv2::Gicv2, pl011::Pl011}; use crate::memory::{GuestMemory, MemoryError}; -use volt_target::aarch64::decode::SignExtend; +use volt_isa_aarch64::decode::SignExtend; #[derive(Debug)] pub enum Error { @@ -487,7 +487,7 @@ impl Machine { continue; } if let crate::aarch64::compile::Error::Decode { word, pc } = &error - && volt_target::aarch64::decode::is_undefined(*word) + && volt_isa_aarch64::decode::is_undefined(*word) { // An Undefined Instruction exception (EC 0, 32-bit instruction). self.cpu.pc = *pc; diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index e1ec209..e13845a 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -767,14 +767,16 @@ fn instruction_diagnostic( let mut bytes = [0; 4]; machine.memory.read(physical, &mut bytes).ok()?; let word = u32::from_le_bytes(bytes); - let decoded = host::dispatch(word); + let decoded = volt_isa_aarch64::decode::decode(word); last = Some(InstructionDiagnostic { virtual_address: address, physical_address: physical, word, decoded: decoded.is_ok(), }); - if !stalled || decoded.is_err() || decoded.is_ok_and(|instruction| instruction.terminates()) + if !stalled + || decoded.is_err() + || decoded.is_ok_and(|instruction| host::ends_block(&instruction, false)) { break; } diff --git a/src/emulator/src/user/mod.rs b/src/emulator/src/user/mod.rs index 35d2e87..4682c6f 100644 --- a/src/emulator/src/user/mod.rs +++ b/src/emulator/src/user/mod.rs @@ -10,12 +10,12 @@ pub mod syscalls; pub use memory::Space; use crate::aarch64::cpu::{Exclusive, Trap}; -use crate::aarch64::host::{Dispatch, dispatch}; +use crate::aarch64::host::run as run_host; use crate::aarch64::{Cache, Cpu}; use crate::memory::{GuestMemory, MemoryError}; use std::ffi::OsString; use std::path::Path; -use volt_target::aarch64::decode::{Instruction, SignExtend, SystemRegister}; +use volt_isa_aarch64::decode::{Instruction, SignExtend, SystemRegister, decode}; #[derive(Debug)] pub enum Error { @@ -110,7 +110,7 @@ pub fn run(path: &Path, args: &[OsString], env: &[(OsString, OsString)]) -> Resu Trap::SimdStruct => simd_struct(&mut space, &mut cpu)?, Trap::Host => { let word = cpu.value as u32; - crate::aarch64::host::run(&mut cpu, word); + run_host(&mut cpu, word); cpu.trap = Trap::None; } Trap::Svc => { @@ -151,19 +151,17 @@ fn fetch( } } let word = u32::from_le_bytes(bytes); - let dispatched = - dispatch(word).map_err(|_| crate::aarch64::compile::Error::Decode { word, pc: address })?; - if let Dispatch::Guest(instruction) = &dispatched { - if !user_instruction(instruction) { + let instruction = + decode(word).map_err(|_| crate::aarch64::compile::Error::Decode { word, pc: address })?; + if !user_instruction(&instruction) { + return Err(crate::aarch64::compile::Error::InvalidUserInstruction); + } + if let Instruction::CacheOp(sys) = &instruction { + // The decoder groups EL0 and privileged cache operations together. + // Only IC IVAU and DC CVAC/CVAU/CIVAC are permitted in userspace. + if sys.op1 != 3 || sys.op2 != 1 || !matches!(sys.crm, 5 | 10 | 11 | 14) { return Err(crate::aarch64::compile::Error::InvalidUserInstruction); } - if let Instruction::CacheOp(sys) = instruction { - // The decoder groups EL0 and privileged cache operations together. - // Only IC IVAU and DC CVAC/CVAU/CIVAC are permitted in userspace. - if sys.op1 != 3 || sys.op2 != 1 || !matches!(sys.crm, 5 | 10 | 11 | 14) { - return Err(crate::aarch64::compile::Error::InvalidUserInstruction); - } - } } if into.len() < 4 { return Err(crate::aarch64::compile::Error::InvalidInstructionLength); diff --git a/src/isa/aarch64/Cargo.toml b/src/isa/aarch64/Cargo.toml new file mode 100644 index 0000000..74e984b --- /dev/null +++ b/src/isa/aarch64/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "volt-isa-aarch64" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +path = "src/lib.rs" + +[lints] +workspace = true diff --git a/src/target/src/aarch64/decode.rs b/src/isa/aarch64/src/decode.rs similarity index 67% rename from src/target/src/aarch64/decode.rs rename to src/isa/aarch64/src/decode.rs index 65ac6e6..1bff71a 100644 --- a/src/target/src/aarch64/decode.rs +++ b/src/isa/aarch64/src/decode.rs @@ -1,4 +1,5 @@ -use super::simd_struct::{Shape, StructDesc}; +use crate::simd_struct::{Shape, StructDesc}; +use crate::{fp, gxf, pauth, simd}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DecodeError { @@ -124,15 +125,61 @@ pub struct Arithmetic { pub width: Width, pub rn: u8, pub rd: u8, - pub immediate: u32, + /// The 12-bit immediate field as encoded, before `shifted` applies. + pub immediate: u16, + /// `lsl #12` on the immediate: `add x0, x1, #0, lsl #12` differs from `add x0, x1, #0`. + pub shifted: bool, +} +impl Arithmetic { + /// The operand value: the immediate field, shifted left by 12 when `shifted`. + pub fn operand(self) -> u32 { + u32::from(self.immediate) << if self.shifted { 12 } else { 0 } + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Addressing { + /// `[xn, #imm]` with the scaled unsigned 12-bit offset (`ldr`, `str`, `prfm`) or, for + /// pairs, the scaled signed 7-bit one (`ldp`, `stp`, `ldnp`, `stnp`). Offset(i64), - Register { rm: u8, extend: Extend, amount: u8 }, + /// `[xn, #simm]` with the unscaled signed 9-bit offset: `ldur`, `stur`, `prfum`, + /// `ldapur`, `stlur`. The offset may be representable as `Offset` too. + Unscaled(i64), + /// `[xn, #simm]` of `ldtr`/`sttr`: the unscaled signed 9-bit offset, but the + /// access is checked as an unprivileged (EL0) one. + Unprivileged(i64), + /// `amount` is `None` when the S bit is clear and `Some(n)` when the offset register + /// is shifted by `n`: for a byte access `Some(0)` is spelled `lsl #0`. + Register { + rm: u8, + extend: Extend, + amount: Option, + }, PreIndex(i64), PostIndex(i64), } +/// The acquire/release class of an `Offset(0)` or `Unscaled` access that is neither a +/// plain load or store nor an exclusive. `ldar`, `stlr` and `ldapr` are the ARMv8 +/// RCsc/RCpc pair; the `L` forms are the LORegion limited-ordering ones. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Ordered { + /// `ldar`: load-acquire. + Acquire, + /// `ldapr`, `ldapur`: load-acquire with only RCpc (processor consistent) ordering. + AcquirePc, + /// `ldlar`: limited-ordering load-acquire. + LimitedAcquire, + /// `stlr`, `stlur`: store-release. + Release, + /// `stllr`: limited-ordering store-release. + LimitedRelease, +} +/// The acquire (`a`) and release (`l`) bits of an exclusive or LSE atomic, spelled +/// `ldaxr`, `stlxr`, `casal`, `ldaddl`, ... +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MemoryOrder { + pub acquire: bool, + pub release: bool, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Memory { pub op: MemoryOp, @@ -141,6 +188,9 @@ pub struct Memory { pub rt: u8, pub addressing: Addressing, pub signed: SignExtend, + /// `Some` for `ldar`, `stlr`, `ldapr`, `ldlar`, `stllr` and the unscaled `ldapur`, + /// `stlur`, `ldapurs*`; the machines run them as the plain access. + pub ordered: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Pair { @@ -151,6 +201,8 @@ pub struct Pair { pub rt2: u8, pub addressing: Addressing, pub signed: SignExtend, + /// `ldnp`, `stnp`: the non-temporal hint form (always `Addressing::Offset`). + pub non_temporal: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Multiply { @@ -189,7 +241,10 @@ pub struct Literal { pub size: Size, pub rt: u8, pub offset: i64, + /// `ldrsw` (literal) sign-extends its word to 64 bits. + pub signed: SignExtend, } +/// `LDXR`, `LDAXR`, `STXR`, `STLXR` and their byte and halfword forms. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Exclusive { pub op: MemoryOp, @@ -197,10 +252,13 @@ pub struct Exclusive { pub rn: u8, pub rt: u8, pub rs: u8, + /// `ldaxr` acquires, `stlxr` releases. The machines ignore it. + pub order: MemoryOrder, } -/// Operation of an LSE atomic. Acquire and release ordering are not modelled -/// separately: each host read-modify-write is sequentially consistent. +/// Operation of an LSE atomic or exclusive pair. The acquire and release bits are kept +/// in `Atomic::order`; the machines do not model them: each host read-modify-write is +/// sequentially consistent. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] pub enum AtomicKind { @@ -232,6 +290,8 @@ pub struct Atomic { pub rt: u8, /// The second register of `LoadPair` and `StorePair`. pub rt2: u8, + /// The `a` and `l` suffixes of `casal`, `ldaddal`, `swpa`, `ldaxp`, `stlxp`, ... + pub order: MemoryOrder, } impl AtomicKind { /// Number of adjacent words the operation reads and writes. @@ -323,6 +383,14 @@ pub struct Prefetch { pub rn: u8, pub addressing: Addressing, } +/// `RPRFM , Xm, [Xn]`: a range prefetch. It is a hint and runs as a no-op. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RangePrefetch { + /// The six-bit range prefetch operation. + pub op: u8, + pub rm: u8, + pub rn: u8, +} /// `PRFM` (literal): a prefetch from a PC-relative address. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct PrefetchLiteral { @@ -332,8 +400,10 @@ pub struct PrefetchLiteral { /// How a modified-immediate instruction combines its immediate with `rd`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ImmCombine { - /// `movi`, `mvni`, `fmov`: replace the register. + /// `movi`, `fmov`: replace the register. Set, + /// `mvni`: replace the register with the (already inverted) pattern. + SetInverted, /// `orr`: `rd |= imm`. Or, /// `bic`: `rd &= !imm`. @@ -342,13 +412,39 @@ pub enum ImmCombine { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdImmediate { pub rd: u8, + /// The encoded 8-bit immediate (`abc:defgh`). + pub imm8: u8, + /// The lane layout and shift the encoding (`cmode`, `op`) selects. + pub form: ImmForm, /// The expanded immediate: the low and high 64 bits (the high half is zero for /// a 64-bit `q == false` form, which also clears the register's upper half). + /// Inverted for `mvni`. pub low: u64, pub high: u64, pub combine: ImmCombine, pub q: bool, } +/// The lane layout of a modified immediate (`AdvSIMDExpandImm` `cmode` classes). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ImmForm { + /// `.8b`/`.16b`: `imm8` in every byte. + Byte, + /// `.4h`/`.8h`: `imm8 << shift` with `shift` 0 or 8 (`lsl`). + Half { shift: u8 }, + /// `.2s`/`.4s`: `imm8 << shift` with `shift` 0, 8, 16 or 24 (`lsl`). + Word { shift: u8 }, + /// `.2s`/`.4s`: `imm8 << shift` with the vacated low bits set, `shift` 8 or 16 + /// (`msl`). + WordMask { shift: u8 }, + /// `movi d`/`.2d`: each bit of `imm8` stands for a 0x00 or 0xff byte. + Double, + /// `fmov .4h`/`.8h`. + FmovHalf, + /// `fmov .2s`/`.4s`. + FmovSingle, + /// `fmov .2d`. + FmovDouble, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdPair { pub op: MemoryOp, @@ -357,6 +453,18 @@ pub struct SimdPair { pub rt: u8, pub rt2: u8, pub addressing: Addressing, + /// `ldnp`, `stnp`: the non-temporal hint form (always `Addressing::Offset`). + pub non_temporal: bool, +} +/// How a [`SimdMemory`] is spelled. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SimdAccess { + /// `ldr`/`str` (`ldur`/`stur` with an unscaled offset) of a `b`, `h`, `s`, `d` or `q` + /// register. + Register, + /// One-register `ld1`/`st1` of a whole `.8b`..`.2d` vector whose elements are `esize` + /// bytes. It moves the same bytes as the `d`/`q` register form. + Ld1 { esize: u8 }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdMemory { @@ -365,6 +473,7 @@ pub struct SimdMemory { pub rn: u8, pub rt: u8, pub addressing: Addressing, + pub access: SimdAccess, } /// How a SIMD structure access updates its base register afterwards. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -401,7 +510,10 @@ pub struct SimdExt { pub rd: u8, pub rn: u8, pub rm: u8, + /// The byte index into the `rm:rn` pair, below 8 when `q` is clear. pub imm: u8, + /// The 128-bit form (`.16b`); clear for `.8b`. + pub q: bool, } /// `dup` from an element of a vector register: replicate lane `index` of /// `rn` across every lane of the destination. @@ -450,6 +562,9 @@ pub struct SimdFmov { pub to_fp: bool, /// True for 64-bit (`x`/`d`), false for 32-bit (`w`/`s`). pub double: bool, + /// The upper doubleword of the vector (`Vd.D[1]`); only with `double`. A move + /// to it keeps the lower doubleword. + pub upper: bool, } /// Three-registers-same integer vector operation (`U`, `size` and `opcode` /// fields verified against `llvm-mc` encodings). @@ -647,9 +762,18 @@ pub enum SystemRegister { CntpCvalEl0, ActlrEl1, Nzcv, - RazWi, + /// Read-as-zero, write-ignored debug register `DBG{B,W}{V,C}R_EL1` (`op0` 2, `op2` 4 to 7: + /// `DBGBVR`, `DBGBCR`, `DBGWVR`, `DBGWCR`). + RazWi { + crm: u8, + op2: u8, + }, /// Read-as-zero, write-ignored: Apple IMP-DEF status registers (`op0` 3). - AppleZero, + AppleZero { + op1: u8, + crm: u8, + op2: u8, + }, /// Apple IMP-DEF configuration register the guest reads back, stored in /// `System::apple[slot]`. Apple(u8), @@ -701,6 +825,8 @@ pub struct System { pub read: bool, pub register: SystemRegister, pub rt: u8, + /// The `#imm` of `msr , #imm` (`DaifSet`, `DaifClear`, `SpSelSet`, + /// `PanSet`); zero for the register forms. pub immediate: u8, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -748,10 +874,21 @@ pub struct Call { pub target: i64, pub link: bool, } +/// Which register branch an [`Indirect`] is. `RET` is architecturally a `BR` with a +/// return hint, but analysis (and branch prediction) must tell them apart. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IndirectKind { + /// `BR Xn`. + Jump, + /// `BLR Xn`: also writes the return address to `X30`. + Call, + /// `RET {Xn}`. + Return, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Indirect { pub rn: u8, - pub link: bool, + pub kind: IndirectKind, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Variable { @@ -870,6 +1007,7 @@ pub enum Instruction { Hint(u8), Prefetch(Prefetch), PrefetchLiteral(PrefetchLiteral), + RangePrefetch(RangePrefetch), Dsb(u8), Dmb(u8), Isb(u8), @@ -893,10 +1031,6 @@ pub enum Instruction { SimdStruct(SimdStruct), SimdCompareZero(SimdCompareZero), SimdFmov(SimdFmov), - SimdMove { - rd: u8, - rn: u8, - }, SimdDup { rd: u8, rn: u8, @@ -913,58 +1047,15 @@ pub enum Instruction { Svc(u16), Psci, Wfi, + /// Scalar floating point; see [`fp`]. + Fp(fp::Op), + /// Integer Advanced SIMD instructions without a structured form; see [`simd`]. + Simd(simd::Op), + /// Pointer authentication; see [`pauth`]. + Pauth(pauth::Op), + /// Guarded execution; see [`gxf`]. + Gxf(gxf::Op), } -impl Instruction { - /// Whether this instruction ends a translated block. With `inline_memory`, - /// plain loads and stores are compiled with an inline fast path and no - /// longer end the block (their slow path exits mid-block instead). - pub fn terminates_with(&self, inline_memory: bool) -> bool { - if inline_memory - && matches!( - self, - Instruction::Memory(_) - | Instruction::Pair(_) - | Instruction::Literal(_) - | Instruction::SimdMemory(_) - | Instruction::SimdPair(_) - ) - { - return false; - } - self.terminates() - } - pub fn terminates(&self) -> bool { - use Instruction::*; - matches!( - self, - Memory(_) - | Exclusive(_) - | Atomic(_) - | Literal(_) - | SimdLiteral(_) - | Pair(_) - | SimdPair(_) - | SimdMemory(_) - | SimdStruct(_) - | B(_) - | BCond(_) - | TestBranch(_) - | Call(_) - | Indirect(_) - | Svc(_) - | Psci - | Wfi - | Wfe - | Brk(_) - | Isb(_) - | DcZva(_) - | AddressTranslate(_) - | Eret - | Tlbi(_) - ) - } -} - /// Whether `word` is architecturally UNDEFINED on this CPU, as opposed to an /// instruction the emulator does not implement. Execution takes an Undefined /// Instruction exception (XNU's `TRAP_DEBUGGER` word `0xe7ffdeff` relies on it). These @@ -1030,6 +1121,45 @@ fn extension(n: u32) -> Extend { Extend::Sxtx, ][(n & 7) as usize] } +/// The `[xn, rm{, ext{ #amount}}]` address of a register-offset load, store or prefetch. +/// `scale` is log2 of the access size, the amount applied when the S bit is set. Options +/// 0, 1, 4 and 5 (the byte and halfword extensions) and a non-zero bits 11:10 are +/// unallocated. +fn register_offset(word: u32, scale: u32) -> Result { + let option = (word >> 13) & 7; + if option & 2 == 0 || word & 0x0c00 != 0x0800 { + return Err(DecodeError::UnsupportedInstruction); + } + Ok(Addressing::Register { + rm: reg(word, 16), + extend: extension(option), + amount: (word & 0x1000 != 0).then_some(scale as u8), + }) +} +/// The address of a load/store-register word without the unsigned-offset or register +/// form: bits 11:10 select unscaled, post-index, unprivileged or pre-index with the +/// signed 9-bit offset. The SIMD&FP forms have no unprivileged variant. +fn immediate_offset(word: u32, vector: bool) -> Result { + let offset = sign_extend(word >> 12, 9); + match (word >> 10) & 3 { + 0 => Ok(Addressing::Unscaled(offset)), + 1 => Ok(Addressing::PostIndex(offset)), + 2 if !vector => Ok(Addressing::Unprivileged(offset)), + 3 => Ok(Addressing::PreIndex(offset)), + _ => Err(DecodeError::UnsupportedInstruction), + } +} +/// The sign extension of a general-register load or store: `opc` is bits 23:22 and +/// `size` the access size. `opc` 0 stores and 1 loads zero-extended; 2 is `ldrs[bhw] Xt` +/// and 3 is `ldrs[bh] Wt`. +fn load_extension(opc: u32, size: Size) -> Result { + match opc { + 0 | 1 => Ok(SignExtend::None), + 2 if size != Size::Double => Ok(SignExtend::To64), + 3 if size == Size::Byte || size == Size::Half => Ok(SignExtend::To32), + _ => Err(DecodeError::UnsupportedInstruction), + } +} fn sign_extend(n: u32, bits: u32) -> i64 { ((n << (32 - bits)) as i32 >> (32 - bits)) as i64 } @@ -1075,9 +1205,8 @@ fn bitmask(word: u32, width: Width) -> Result { Ok(immediate) } /// Decode an Advanced SIMD structure load/store word (`word & 0xbe00_0000 == -/// 0x0c00_0000`) into its descriptor and post-index, or `None` for the -/// whole-register LD1/ST1 forms that `SimdMemory` handles. -fn simd_struct(word: u32) -> Result, DecodeError> { +/// 0x0c00_0000`) into its descriptor and post-index. +fn simd_struct(word: u32) -> Result<(StructDesc, StructPost), DecodeError> { use DecodeError::UnsupportedInstruction as Bad; let q = word & 0x4000_0000 != 0; let rm = reg(word, 16); @@ -1118,9 +1247,6 @@ fn simd_struct(word: u32) -> Result, DecodeErro if desc.interleaved && !q && size == 3 { return Err(Bad); } - if desc.registers == 1 && !matches!(post, StructPost::Register(_)) { - return Ok(None); - } } else { // Bits 15:13 give the element scale (and, with R, the structure count); // the lane index is spread over Q, S and size. @@ -1139,7 +1265,7 @@ fn simd_struct(word: u32) -> Result, DecodeErro }; (desc.shape, desc.esize, desc.lane) = (shape, esize, lane); } - Ok(Some((desc, post))) + Ok((desc, post)) } /// Number of Apple IMP-DEF configuration registers `System::apple` holds. pub const APPLE_SLOTS: usize = 21; @@ -1194,6 +1320,11 @@ fn apple_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option Apple(0), (1, 1, 0) => Apple(1), @@ -1202,7 +1333,7 @@ fn apple_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option Apple(4), (1, 5, 0) => Apple(5), (1, 6, 0) => Apple(6), - (1, 13, 0) => AppleZero, + (1, 13, 0) => zero, (2, 0, 0) => Apple(7), (2, 1, 0) => Apple(8), (2, 2, 0) => Apple(9), @@ -1215,14 +1346,14 @@ fn apple_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option Apple(16), (5, 0, 0) => Apple(17), (5, 0, 1) => Apple(18), - (5, 1, 1) => AppleZero, + (5, 1, 1) => zero, (5, 3, 1) => Apple(19), (7, 0, 4) => Apple(20), - (7, 6, 4) => AppleZero, + (7, 6, 4) => zero, // The S3_5_c15 state registers the kernel reads during early boot. None are named // in the XNU source tree that is available here, so they read as zero and writes // are dropped until their roles are identified. - (5, 2..=7, 0) => AppleZero, + (5, 2..=7, 0) => zero, // The EL2 and guarded-level banks the SPTM monitor configures: stored, so a // read-modify-write or a read-back sees what was written. (4..=6, _, _) => AppleBank(apple_bank_slot(op1, crm, op2) as u16), @@ -1264,7 +1395,10 @@ fn system_register( ) -> Result { use SystemRegister::*; if op0 == 2 && op1 == 0 && crn == 0 && (4..=7).contains(&op2) { - return Ok(RazWi); + return Ok(RazWi { + crm: crm as u8, + op2: op2 as u8, + }); } if let Some(register) = apple_register(op0, op1, crn, crm, op2) { return Ok(register); @@ -1381,6 +1515,14 @@ fn system_register( }) } +/// The 16-bit pattern of the 8-bit floating-point immediate `imm8` (`VFPExpandImm` +/// for half precision), as the vector `fmov` takes it. +pub fn expand_half_immediate(imm8: u64) -> u64 { + let sign = imm8 >> 7; + let b = (imm8 >> 6) & 1; + (sign << 15) | ((b ^ 1) << 14) | (b * 0b11 << 12) | ((imm8 & 0x3f) << 6) +} + /// The 32- or 64-bit pattern of the 8-bit floating-point immediate `imm8` /// (`VFPExpandImm`), as `fmov` and the vector `fmov` take it. pub fn expand_immediate(imm8: u64, double: bool) -> u64 { @@ -1394,7 +1536,28 @@ pub fn expand_immediate(imm8: u64, double: bool) -> u64 { } } +/// Decode one instruction word. +/// +/// The families in [`fp`], [`simd`], [`pauth`] and [`gxf`] are tried first and take +/// precedence over the structured forms of this module wherever both recognise a word: +/// the emulator executes those families from their decoded form, never by lifting them. pub fn decode(word: u32) -> Result { + if let Some(op) = fp::decode(word) { + return Ok(Instruction::Fp(op)); + } + if let Some(op) = simd::decode(word) { + return Ok(Instruction::Simd(op)); + } + if let Some(op) = pauth::decode(word) { + return Ok(Instruction::Pauth(op)); + } + if let Some(op) = gxf::decode(word) { + return Ok(Instruction::Gxf(op)); + } + structured(word) +} + +fn structured(word: u32) -> Result { use DecodeError::UnsupportedInstruction; use Instruction::*; let width = width(word); @@ -1559,9 +1722,20 @@ pub fn decode(word: u32) -> Result { if word & 0x001f_0000 != 0x001f_0000 || word & 0x0000_fc1f != 0 { return Err(UnsupportedInstruction); } - return match (word >> 21) & 7 { - 0 | 2 => Ok(Indirect(self::Indirect { rn, link: false })), - 1 => Ok(Indirect(self::Indirect { rn, link: true })), + // opc is bits 24:21; bit 24 set is the authenticated forms (`braa`, `retaa`, ...). + return match (word >> 21) & 15 { + 0 => Ok(Indirect(self::Indirect { + rn, + kind: IndirectKind::Jump, + })), + 1 => Ok(Indirect(self::Indirect { + rn, + kind: IndirectKind::Call, + })), + 2 => Ok(Indirect(self::Indirect { + rn, + kind: IndirectKind::Return, + })), 4 if rn == 31 => Ok(Eret), _ => Err(UnsupportedInstruction), }; @@ -1576,7 +1750,8 @@ pub fn decode(word: u32) -> Result { width, rn, rd, - immediate: ((word >> 10) & 0xfff) << if word & 0x0040_0000 != 0 { 12 } else { 0 }, + immediate: ((word >> 10) & 0xfff) as u16, + shifted: word & 0x0040_0000 != 0, })); } // PRFM (unsigned offset), PRFUM (unscaled offset) and PRFM (register). They @@ -1592,18 +1767,28 @@ pub fn decode(word: u32) -> Result { return Ok(Prefetch(self::Prefetch { op: rd, rn, - addressing: Addressing::Offset(sign_extend(word >> 12, 9)), + addressing: Addressing::Unscaled(sign_extend(word >> 12, 9)), })); } if word & 0xffe0_0c00 == 0xf8a0_0800 { + let addressing = register_offset(word, 3)?; + // RPRFM takes the PRFM (register) slots whose Rt is 0b11xxx; its 6-bit operation + // is `option<2>:option<0>:S:Rt<2:0>`. + if rd >= 24 { + let option = (word >> 13) & 7; + return Ok(RangePrefetch(self::RangePrefetch { + op: ((option >> 2) << 5 + | (option & 1) << 4 + | ((word >> 12) & 1) << 3 + | u32::from(rd & 7)) as u8, + rm, + rn, + })); + } return Ok(Prefetch(self::Prefetch { op: rd, rn, - addressing: Addressing::Register { - rm, - extend: extension(word >> 13), - amount: if word & 0x1000 != 0 { 3 } else { 0 }, - }, + addressing, })); } // PRFM (literal). @@ -1614,15 +1799,18 @@ pub fn decode(word: u32) -> Result { })); } if word & 0x3f00_0000 == 0x1800_0000 { - let size = match word >> 30 { - 0 => Size::Word, - 1 => Size::Double, + let (size, signed) = match word >> 30 { + 0 => (Size::Word, SignExtend::None), + 1 => (Size::Double, SignExtend::None), + // LDRSW (literal). + 2 => (Size::Word, SignExtend::To64), _ => return Err(UnsupportedInstruction), }; return Ok(Literal(self::Literal { size, rt: rd, offset: sign_extend(word >> 5, 19) << 2, + signed, })); } // LDR (literal, SIMD&FP): S, D or Q register from a PC-relative address. @@ -1640,9 +1828,14 @@ pub fn decode(word: u32) -> Result { })); } // LSE atomics (ARMv8.1): LDADD/LDCLR/LDEOR/LDSET/LDSMAX/LDSMIN/LDUMAX/LDUMIN and - // SWP share one encoding, split by o3 (bit 15) and opc (bits 14:12). LDAPR, the - // RCpc load, is the o3=1, opc=100 slot with Rs unused. + // SWP share one encoding, split by o3 (bit 15) and opc (bits 14:12); A (bit 23) is + // acquire and R (bit 22) release. LDAPR, the RCpc load, is the A=1, R=0, o3=1, + // opc=100 slot with Rs unused. if word & 0x3f20_0c00 == 0x3820_0000 { + let order = MemoryOrder { + acquire: word & 0x0080_0000 != 0, + release: word & 0x0040_0000 != 0, + }; let kind = match (word & 0x8000 != 0, (word >> 12) & 7) { (false, 0) => AtomicKind::Add, (false, 1) => AtomicKind::Clr, @@ -1653,7 +1846,7 @@ pub fn decode(word: u32) -> Result { (false, 6) => AtomicKind::UMax, (false, 7) => AtomicKind::UMin, (true, 0) => AtomicKind::Swap, - (true, 4) if rm == 31 => { + (true, 4) if rm == 31 && order.acquire && !order.release => { return Ok(Memory(self::Memory { op: MemoryOp::Load, size: size(word), @@ -1661,6 +1854,7 @@ pub fn decode(word: u32) -> Result { rt: rd, addressing: Addressing::Offset(0), signed: SignExtend::None, + ordered: Some(Ordered::AcquirePc), })); } _ => return Err(UnsupportedInstruction), @@ -1672,9 +1866,15 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rt2: 0, + order, })); } // CAS/CASP (ARMv8.1): compare with Rs (the pair Rs, Rs+1 for CASP); store Rt on a match. + // L (bit 22) is acquire and o0 (bit 15) release. + let cas_order = MemoryOrder { + acquire: word & 0x0040_0000 != 0, + release: word & 0x8000 != 0, + }; if word & 0x3fa0_7c00 == 0x08a0_7c00 { return Ok(Atomic(self::Atomic { kind: AtomicKind::Cas, @@ -1683,6 +1883,7 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rt2: 0, + order: cas_order, })); } if word & 0xbfa0_7c00 == 0x0820_7c00 { @@ -1702,13 +1903,20 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rt2: 0, + order: cas_order, })); } if word & 0x3f00_0000 == 0x0800_0000 { + // o0 (bit 15): release for a store, acquire for a load. + let load = word & 0x0040_0000 != 0; + let o0 = word & 0x8000 != 0; + let order = MemoryOrder { + acquire: load && o0, + release: !load && o0, + }; if word & 0x0020_0000 != 0 { // LDXP/LDAXP/STXP/STLXP: a pair of words or doublewords; the load form has // no status register. - let load = word & 0x0040_0000 != 0; if word & 0x8000_0000 == 0 || word & 0x0080_0000 != 0 || (load && rm != 31) { return Err(UnsupportedInstruction); } @@ -1727,10 +1935,10 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rt2: reg(word, 10), + order, })); } let size = size(word); - let load = word & 0x0040_0000 != 0; let op = if load { MemoryOp::Load } else { @@ -1741,7 +1949,8 @@ pub fn decode(word: u32) -> Result { return Err(UnsupportedInstruction); } if word & 0x0080_0000 != 0 { - if rs != 31 || (load && word & 0x8000 == 0) { + // LDAR/STLR (o0 set) and the LORegion LDLAR/STLLR (o0 clear). + if rs != 31 { return Err(UnsupportedInstruction); } return Ok(Memory(self::Memory { @@ -1751,6 +1960,12 @@ pub fn decode(word: u32) -> Result { rt: rd, addressing: Addressing::Offset(0), signed: SignExtend::None, + ordered: Some(match (load, o0) { + (true, true) => Ordered::Acquire, + (true, false) => Ordered::LimitedAcquire, + (false, true) => Ordered::Release, + (false, false) => Ordered::LimitedRelease, + }), })); } if load && rs != 31 { @@ -1762,46 +1977,59 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rs, + order, })); } - // LD1..LD4 / ST1..ST4 (multiple and single structures) and LD1R..LD4R. - // A single-register LD1/ST1 without a register post-index is left to the - // `SimdMemory` blocks below, which the JIT inlines. - if word & 0xbe00_0000 == 0x0c00_0000 - && let Some((desc, post)) = simd_struct(word)? - { + // LD1..LD4 / ST1..ST4 (multiple and single structures) and LD1R..LD4R. A single + // register without a register post-index moves a whole vector like `ldr d`/`ldr q`, + // which the JIT inlines: that form is a `SimdMemory` that remembers its arrangement. + if word & 0xbe00_0000 == 0x0c00_0000 { + let (desc, post) = simd_struct(word)?; + if desc.shape == Shape::Multiple + && desc.registers == 1 + && !matches!(post, StructPost::Register(_)) + { + let bytes = desc.vector_bytes(); + return Ok(SimdMemory(self::SimdMemory { + op: if desc.store { + MemoryOp::Store + } else { + MemoryOp::Load + }, + bytes, + rn, + rt: desc.rt, + addressing: if post == StructPost::Immediate { + Addressing::PostIndex(i64::from(bytes)) + } else { + Addressing::Offset(0) + }, + access: SimdAccess::Ld1 { esize: desc.esize }, + })); + } return Ok(SimdStruct(self::SimdStruct { desc, rn, post })); } - // LD1/ST1 (one structure): a full or half-width SIMD register. - // - // bits 29:24 are `001100`, bits 14:12 are `111`, bit 23 selects the - // no-offset vs post-index addressing; bits 21:16 are the `size:opc` - // structure field, which must be 0b000000 or 0b011111. - if word & 0xbf00_8000 == 0x0c00_0000 && word & 0x0000_7000 == 0x0000_7000 { - let bytes = if word & 0x4000_0000 != 0 { 16 } else { 8 }; - let post_index = word & 0x0080_0000 != 0; - let index = (word >> 16) & 0x3f; - let addressing = if post_index { - if index != 31 { - return Err(UnsupportedInstruction); - } - Addressing::PostIndex(bytes as i64) - } else { - if index != 0 { - return Err(UnsupportedInstruction); - } - Addressing::Offset(0) - }; - return Ok(SimdMemory(self::SimdMemory { - op: if word & 0x0040_0000 != 0 { - MemoryOp::Load - } else { + // LDAPUR/STLUR and LDAPURS* (FEAT_LRCPC2): the RCpc/release accesses with an unscaled + // offset. + if word & 0x3f20_0c00 == 0x1900_0000 { + let opc = (word >> 22) & 3; + let size = size(word); + return Ok(Memory(self::Memory { + op: if opc == 0 { MemoryOp::Store + } else { + MemoryOp::Load }, - bytes: bytes as u8, + size, rn, rt: rd, - addressing, + addressing: Addressing::Unscaled(sign_extend(word >> 12, 9)), + signed: load_extension(opc, size)?, + ordered: Some(if opc == 0 { + Ordered::Release + } else { + Ordered::AcquirePc + }), })); } if word & 0x3a00_0000 == 0x3800_0000 { @@ -1825,33 +2053,9 @@ pub fn decode(word: u32) -> Result { let addressing = if word & 0x0100_0000 != 0 { Addressing::Offset((((word >> 10) & 0xfff) * bytes as u32) as i64) } else if word & 0x0020_0000 != 0 { - if word & 0x0000_0c00 != 0x800 { - return Err(UnsupportedInstruction); - } - let extend = match (word >> 13) & 7 { - 2 => Extend::Uxtw, - 3 => Extend::Uxtx, - 6 => Extend::Sxtw, - 7 => Extend::Sxtx, - _ => return Err(UnsupportedInstruction), - }; - Addressing::Register { - rm, - extend, - amount: if word & 0x1000 != 0 { - (bytes as u32).trailing_zeros() as u8 - } else { - 0 - }, - } + register_offset(word, (bytes as u32).trailing_zeros())? } else { - let offset = sign_extend(word >> 12, 9); - match (word >> 10) & 3 { - 0 | 2 => Addressing::Offset(offset), - 1 => Addressing::PostIndex(offset), - 3 => Addressing::PreIndex(offset), - _ => return Err(UnsupportedInstruction), - } + immediate_offset(word, true)? }; return Ok(SimdMemory(self::SimdMemory { op: if is_load { @@ -1863,46 +2067,18 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, addressing, + access: SimdAccess::Register, })); } let opc = (word >> 22) & 3; let size = size(word); - let signed = match opc { - 0 | 1 => SignExtend::None, - 2 if size != Size::Double => SignExtend::To64, - 3 if size == Size::Byte || size == Size::Half => SignExtend::To32, - _ => return Err(UnsupportedInstruction), - }; + let signed = load_extension(opc, size)?; let addressing = if word & 0x0100_0000 != 0 { Addressing::Offset((((word >> 10) & 0xfff) * size as u32) as i64) } else if word & 0x0020_0000 != 0 { - if word & 0x0000_0c00 != 0x800 { - return Err(UnsupportedInstruction); - } - let extend = match (word >> 13) & 7 { - 2 => Extend::Uxtw, - 3 => Extend::Uxtx, - 6 => Extend::Sxtw, - 7 => Extend::Sxtx, - _ => return Err(UnsupportedInstruction), - }; - Addressing::Register { - rm, - extend, - amount: if word & 0x1000 != 0 { - (size as u8).trailing_zeros() as u8 - } else { - 0 - }, - } + register_offset(word, (size as u32).trailing_zeros())? } else { - let offset = sign_extend(word >> 12, 9); - match (word >> 10) & 3 { - 0 | 2 => Addressing::Offset(offset), - 1 => Addressing::PostIndex(offset), - 3 => Addressing::PreIndex(offset), - _ => return Err(UnsupportedInstruction), - } + immediate_offset(word, false)? }; return Ok(Memory(self::Memory { op: if opc == 0 { @@ -1915,6 +2091,7 @@ pub fn decode(word: u32) -> Result { rt: rd, addressing, signed, + ordered: None, })); } if word & 0xffff_f01f == 0xd503_201f { @@ -1990,17 +2167,28 @@ pub fn decode(word: u32) -> Result { | SystemRegister::OslarEl1 | SystemRegister::OsdlrEl1 | SystemRegister::OslsrEl1 - | SystemRegister::RazWi => 2, + | SystemRegister::RazWi { .. } => 2, _ => 3, }; if (word >> 19) & 3 != expected || (read && expected == 0) { return Err(UnsupportedInstruction); } + // `msr , #imm` has no register operand (Rt must be 31), and PAN and + // SPSel take a one-bit immediate; any other word is unallocated. + let immediate = ((word >> 8) & 15) as u8; + let pstate = expected == 0; + if pstate + && (rd != 31 + || (matches!(register, SystemRegister::SpSelSet | SystemRegister::PanSet) + && immediate > 1)) + { + return Err(UnsupportedInstruction); + } return Ok(System(self::System { read, register, rt: rd, - immediate: ((word >> 8) & 15) as u8, + immediate: if pstate { immediate } else { 0 }, })); } // BRK #imm16: opc 001 with LL 00. Its immediate is the whole field, so any value @@ -2067,6 +2255,8 @@ pub fn decode(word: u32) -> Result { let vector = word & 0x0400_0000 != 0; let load = word & 0x0040_0000 != 0; let opc = word >> 30; + // Bits 24:23 are 00 for the no-allocate pair (LDNP, STNP). + let non_temporal = (word >> 23) & 7 == 0; if vector { let bytes = match opc { 0 => 4, // S @@ -2092,11 +2282,12 @@ pub fn decode(word: u32) -> Result { rt: rd, rt2: reg(word, 10), addressing, + non_temporal, })); } let load = word & 0x0040_0000 != 0; let opc = word >> 30; - if opc == 1 && !load { + if opc == 1 && (!load || non_temporal) { return Err(UnsupportedInstruction); } let size = match opc { @@ -2122,6 +2313,7 @@ pub fn decode(word: u32) -> Result { rt: rd, rt2: reg(word, 10), addressing, + non_temporal, signed: if opc == 1 { SignExtend::To64 } else { @@ -2132,38 +2324,64 @@ pub fn decode(word: u32) -> Result { // Advanced SIMD modified immediate: MOVI, MVNI, ORR, BIC and FMOV (vector). // `AdvSIMDExpandImm` selects the lane pattern from `cmode` and `op`. // `immh` (bits 22:19) is zero here; a non-zero `immh` is a shift by immediate. + // Bit 11 (`o2`) is zero except in the half-precision `fmov`. if word & 0x9ff8_0400 == 0x0f00_0400 { let q = word & 0x4000_0000 != 0; let op = word & 0x2000_0000 != 0; + let o2 = word & 0x0000_0800 != 0; let cmode = (word >> 12) & 0xf; - let imm8 = u64::from((((word >> 16) & 7) << 5) | ((word >> 5) & 0x1f)); + let imm8 = (((word >> 16) & 7) << 5) | ((word >> 5) & 0x1f); let rd = (word & 0x1f) as u8; let twice = |v: u64| v | (v << 32); - let imm = match cmode { + let wide = |lane: u64| lane * 0x0001_0001_0001_0001; + let wide_imm = u64::from(imm8); + let (form, pattern) = match cmode { // 32-bit lanes, shifted left by 0, 8, 16 or 24. - 0..=7 => twice(imm8 << (8 * (cmode >> 1))), + 0..=7 if !o2 => { + let shift = 8 * (cmode >> 1); + ( + ImmForm::Word { shift: shift as u8 }, + twice(wide_imm << shift), + ) + } // 16-bit lanes, shifted left by 0 or 8. - 8..=11 => { - let lane = imm8 << (8 * ((cmode >> 1) & 1)); - twice(lane | (lane << 16)) + 8..=11 if !o2 => { + let shift = 8 * ((cmode >> 1) & 1); + ( + ImmForm::Half { shift: shift as u8 }, + wide(wide_imm << shift), + ) } // 32-bit lanes with the vacated low bits set to ones. - 12 => twice((imm8 << 8) | 0xff), - 13 => twice((imm8 << 16) | 0xffff), - // Bytes: imm8 in every byte, or each bit of imm8 as a 0x00/0xff byte. - 14 if !op => imm8 * 0x0101_0101_0101_0101, - 14 => (0..8).fold(0, |acc, bit| { - acc | (if (imm8 >> bit) & 1 != 0 { 0xff } else { 0 }) << (8 * bit) - }), - // FMOV: the 8-bit float immediate widened to f32 (twice) or f64. - _ if !op => twice(expand_immediate(imm8, false)), - _ if q => expand_immediate(imm8, true), + 12 | 13 if !o2 => { + let shift = 8 + 8 * (cmode & 1); + ( + ImmForm::WordMask { shift: shift as u8 }, + twice((wide_imm << shift) | ((1 << shift) - 1)), + ) + } + // Bytes: imm8 in every byte. + 14 if !o2 && !op => (ImmForm::Byte, wide_imm * 0x0101_0101_0101_0101), + // Each bit of imm8 as a 0x00/0xff byte, in the one 64-bit lane. + 14 if !o2 => ( + ImmForm::Double, + (0..8).fold(0, |acc, bit| { + acc | (if (wide_imm >> bit) & 1 != 0 { 0xff } else { 0 }) << (8 * bit) + }), + ), + // FMOV: the 8-bit float immediate widened to f16, f32 (twice) or f64. + 15 if !op && o2 => (ImmForm::FmovHalf, wide(expand_half_immediate(wide_imm))), + 15 if !op => ( + ImmForm::FmovSingle, + twice(expand_immediate(wide_imm, false)), + ), + 15 if q && !o2 => (ImmForm::FmovDouble, expand_immediate(wide_imm, true)), _ => return Err(UnsupportedInstruction), }; // Odd cmode below 12 is ORR/BIC; MOVI becomes MVNI (inverted) with op set. let (value, combine) = if cmode < 12 && cmode & 1 == 1 { ( - imm, + pattern, if op { ImmCombine::AndNot } else { @@ -2171,16 +2389,18 @@ pub fn decode(word: u32) -> Result { }, ) } else if cmode < 14 && op { - (!imm, ImmCombine::Set) + (!pattern, ImmCombine::SetInverted) } else { - (imm, ImmCombine::Set) + (pattern, ImmCombine::Set) }; return Ok(SimdImm(SimdImmediate { rd, - low: value, - high: if q { value } else { 0 }, + imm8: imm8 as u8, + form, combine, q, + low: value, + high: if q { value } else { 0 }, })); } if word & 0xbf3f_fc00 == 0x0e20_9800 { @@ -2204,6 +2424,17 @@ pub fn decode(word: u32) -> Result { rn, to_fp: word & 0x0001_0000 != 0, double: double64, + upper: false, + })); + } + // `fmov Vd.D[1], Xn` / `fmov Xd, Vn.D[1]`: the upper doubleword of the vector. + if word & 0xfffe_fc00 == 0x9eae_0000 { + return Ok(SimdFmov(self::SimdFmov { + rd, + rn, + to_fp: word & 0x0001_0000 != 0, + double: true, + upper: true, })); } // Across-lanes reductions: bits 21:17 = 11000, bits 11:10 = 10. They share @@ -2243,9 +2474,6 @@ pub fn decode(word: u32) -> Result { let q = word & 0x4000_0000 != 0; let u = word & 0x2000_0000 != 0; let size = ((word >> 22) & 3) as u8; - if !q && size == 3 { - return Err(UnsupportedInstruction); - } let opc = ((word >> 11) & 31) as u8; let op = match (u, opc) { (false, 0b10000) => Add, @@ -2298,6 +2526,35 @@ pub fn decode(word: u32) -> Result { (true, 0b00011) if size == 3 => Bif, _ => return Err(UnsupportedInstruction), }; + // `orn` and `bif` use `size` as an opcode; every other 64-bit lane needs + // the full vector and exists only in these groups. + if size == 3 + && !matches!(op, Orn | Bif) + && !(q + && matches!( + op, + Add | Sub + | CmEq + | CmGt + | CmGe + | CmHi + | CmHs + | CmTst + | SSHl + | USHl + | SQShl + | UQShl + | SQRShl + | UQRShl + | SQAdd + | UQAdd + | SQSub + | UQSub + | AddP + )) + { + return Err(UnsupportedInstruction); + } return Ok(SimdAlu(self::SimdAlu { op, rd, @@ -2310,7 +2567,7 @@ pub fn decode(word: u32) -> Result { // Table vector lookup: `tbl` and `tbx` over one to four table registers. // Bits 14:13 hold the table length, bit 12 selects tbl/tbx, and bits 11:10 // are reserved, so the class mask covers everything else. - if word & 0xbf20_8c00 == 0x0e00_0000 { + if word & 0xbfe0_8c00 == 0x0e00_0000 { return Ok(SimdTable(self::SimdTable { rd: (word & 0x1f) as u8, rn: ((word >> 5) & 0x1f) as u8, @@ -2320,29 +2577,33 @@ pub fn decode(word: u32) -> Result { q: word & 0x4000_0000 != 0, })); } - // `mov .[index], ` and the `umov`/`mov` lane-to-general form. - if word & 0xbf80_fc00 == 0x0e00_1c00 || word & 0xbf80_fc00 == 0x0e00_3c00 { + // `mov .[index], ` (`ins`, general): only the 128-bit form exists. + if word & 0xffe0_fc00 == 0x4e00_1c00 { let imm5 = (word >> 16) & 0x1f; if imm5 != 0 { let esize = imm5.trailing_zeros() as u8; - let index = (imm5 >> (esize + 1)) as u8; - let q = word & 0x4000_0000 != 0; - if esize <= 3 && (q || esize < 3) { - return if word & 0x0000_2000 == 0 { - Ok(SimdInsert(self::SimdInsert { - rd, - rn, - esize, - index, - })) - } else { - Ok(SimdExtract(self::SimdExtract { - rd, - rn, - esize, - index, - })) - }; + if esize <= 3 { + return Ok(SimdInsert(self::SimdInsert { + rd, + rn, + esize, + index: (imm5 >> (esize + 1)) as u8, + })); + } + } + } + // `umov`/`mov , .[index]`: `Q` is set exactly for the doubleword lane. + if word & 0xbfe0_fc00 == 0x0e00_3c00 { + let imm5 = (word >> 16) & 0x1f; + if imm5 != 0 { + let esize = imm5.trailing_zeros() as u8; + if esize <= 3 && (word & 0x4000_0000 != 0) == (esize == 3) { + return Ok(SimdExtract(self::SimdExtract { + rd, + rn, + esize, + index: (imm5 >> (esize + 1)) as u8, + })); } } } @@ -2363,7 +2624,7 @@ pub fn decode(word: u32) -> Result { } // `dup` from an element: the imm5 field encodes the element size as the // position of its lowest set bit and the index in the bits above. - if word & 0xbf80_fc00 == 0x0e00_0400 { + if word & 0xbfe0_fc00 == 0x0e00_0400 { let q = word & 0x4000_0000 != 0; let imm5 = (word >> 16) & 0x1f; if imm5 != 0 { @@ -2379,7 +2640,8 @@ pub fn decode(word: u32) -> Result { } } } - if word & 0xbf80_fc00 == 0x0e00_0c00 { + // `dup ., `: the doubleword arrangement needs the full vector. + if word & 0xbfe0_fc00 == 0x0e00_0c00 { let q = (word >> 30) & 1 != 0; let imm5 = (word >> 16) & 0x1f; if imm5 != 0 { @@ -2391,13 +2653,6 @@ pub fn decode(word: u32) -> Result { } } } - if word & 0xbef0_fc00 == 0x0ea0_1c00 { - let rn = ((word >> 5) & 0x1f) as u8; - let rm = ((word >> 16) & 0x1f) as u8; - if rn == rm { - return Ok(SimdMove { rd, rn }); - } - } if word & 0xfc00_0000 == 0x1400_0000 { return Ok(B(sign_extend(word, 26) << 2)); } @@ -2412,11 +2667,16 @@ pub fn decode(word: u32) -> Result { })); } if word & 0xbfe0_8400 == 0x2e00_0000 { - let rm = ((word >> 16) & 0x1f) as u8; + let q = word & 0x4000_0000 != 0; let imm = ((word >> 11) & 0xf) as u8; + // The 64-bit form extracts from a 16-byte pair, so the index is below 8. + if !q && imm >= 8 { + return Err(UnsupportedInstruction); + } + let rm = ((word >> 16) & 0x1f) as u8; let rn = ((word >> 5) & 0x1f) as u8; let rd = (word & 0x1f) as u8; - return Ok(SimdExt(self::SimdExt { rd, rn, rm, imm })); + return Ok(SimdExt(self::SimdExt { rd, rn, rm, imm, q })); } if word & 0x7fff_0000 == 0x5ac0_0000 { let op = match (word >> 10) & 63 { @@ -2518,7 +2778,7 @@ mod tests { Addressing::Register { rm: 6, extend: Extend::Uxtx, - amount: 0, + amount: None, }, ), ( @@ -2529,13 +2789,13 @@ mod tests { Addressing::Register { rm: 6, extend: Extend::Uxtx, - amount: 3, + amount: Some(3), }, ), (0xf8410d07, Size::Double, 8, 7, Addressing::PreIndex(16)), (0xf8410549, Size::Double, 10, 9, Addressing::PostIndex(16)), - (0xf85f818b, Size::Double, 12, 11, Addressing::Offset(-8)), - (0xb85fc1cd, Size::Word, 14, 13, Addressing::Offset(-4)), + (0xf85f818b, Size::Double, 12, 11, Addressing::Unscaled(-8)), + (0xb85fc1cd, Size::Word, 14, 13, Addressing::Unscaled(-4)), ]; for (word, size, rn, rt, addressing) in rows { assert_eq!( @@ -2546,7 +2806,8 @@ mod tests { rn, rt, addressing, - signed: SignExtend::None + signed: SignExtend::None, + ordered: None, })) ); } @@ -2557,8 +2818,9 @@ mod tests { size: Size::Word, rn: 7, rt: 6, - addressing: Addressing::Offset(-4), - signed: SignExtend::None + addressing: Addressing::Unscaled(-4), + signed: SignExtend::None, + ordered: None, })) ); for (word, op, size, rn, rt, rt2, addressing) in [ @@ -2644,7 +2906,9 @@ mod tests { rt, rt2, addressing, - signed: SignExtend::None + signed: SignExtend::None, + // 0xa8300c02 is `stnp x2, x3, [x0, #-256]`; the rest are `ldp`/`stp`. + non_temporal: word == 0xa8300c02, })) ); } @@ -2682,7 +2946,7 @@ mod tests { read: true, register, rt: 0, - immediate: ((word >> 8) & 15) as u8 + immediate: 0 })) ); } @@ -2700,7 +2964,7 @@ mod tests { read: false, register, rt: 0, - immediate: ((word >> 8) & 15) as u8 + immediate: 0 })) ); } @@ -2720,6 +2984,150 @@ mod tests { })) ); } + #[test] + fn arithmetic_immediate_keeps_its_shift() { + let imm = |op, rn, rd, width, immediate, shifted| { + Ok(Instruction::ArithImm(Arithmetic { + op, + flags: false, + width, + rn, + rd, + immediate, + shifted, + })) + }; + // `add x0, x1, #0`, `add x0, x1, #0, lsl #12`, `add sp, x1, #0, lsl #12`. + assert_eq!( + decode(0x91000020), + imm(ArithOp::Add, 1, 0, Width::X64, 0, false) + ); + assert_eq!( + decode(0x91400020), + imm(ArithOp::Add, 1, 0, Width::X64, 0, true) + ); + assert_eq!( + decode(0x9140003f), + imm(ArithOp::Add, 1, 31, Width::X64, 0, true) + ); + // `add w0, w1, #3, lsl #12`, `sub x0, x1, #5, lsl #12`. + assert_eq!( + decode(0x11400c20), + imm(ArithOp::Add, 1, 0, Width::W32, 3, true) + ); + assert_eq!( + decode(0xd1401420), + imm(ArithOp::Sub, 1, 0, Width::X64, 5, true) + ); + let Ok(Instruction::ArithImm(a)) = decode(0x917ffc20) else { + panic!("add x0, x1, #0xfff, lsl #12 decodes"); + }; + assert_eq!((a.immediate, a.operand()), (0xfff, 0xfff000)); + } + + #[test] + fn debug_and_apple_zero_registers_keep_their_coordinates() { + use SystemRegister::*; + for (word, register) in [ + (0xd5300080, RazWi { crm: 0, op2: 4 }), + (0xd53000e0, RazWi { crm: 0, op2: 7 }), + (0xd5300fe0, RazWi { crm: 15, op2: 7 }), + ( + 0xd53df600, + AppleZero { + op1: 5, + crm: 6, + op2: 0, + }, + ), + ( + 0xd53ff680, + AppleZero { + op1: 7, + crm: 6, + op2: 4, + }, + ), + ( + 0xd539fd00, + AppleZero { + op1: 1, + crm: 13, + op2: 0, + }, + ), + ( + 0xd53df120, + AppleZero { + op1: 5, + crm: 1, + op2: 1, + }, + ), + ] { + assert_eq!( + decode(word), + Ok(Instruction::System(System { + read: true, + register, + rt: 0, + immediate: 0 + })), + "{word:#010x}" + ); + } + } + + #[test] + fn authenticated_branch_space_is_not_plain_branches() { + // Opcode bit 24 set with no authentication operand: objdump prints `.inst ...; undefined`. + for word in [0xd71f0000, 0xd73f0000, 0xd75f0000, 0xd79f03e0, 0xd71f03e0] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:#010x}" + ); + } + assert!(matches!(decode(0xd61f0000), Ok(Instruction::Indirect(_)))); + assert!(matches!(decode(0xd69f03e0), Ok(Instruction::Eret))); + } + + #[test] + fn msr_immediate_needs_rt_31_and_a_valid_immediate() { + // `msr pan, #0` and `msr daifset, #0` with Rt = 0, PAN/SPSel with CRm >= 2 (objdump + // spells these `msr s0_..., x0`). + for word in [ + 0xd5004080, 0xd50041a0, 0xd50340c0, 0xd50342e0, 0xd500429f, 0xd50042bf, 0xd50043bf, + 0xd500439f, + ] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:#010x}" + ); + } + for (word, register, immediate) in [ + (0xd500409f, SystemRegister::PanSet, 0), + (0xd500419f, SystemRegister::PanSet, 1), + (0xd50040bf, SystemRegister::SpSelSet, 0), + (0xd50041bf, SystemRegister::SpSelSet, 1), + (0xd50340df, SystemRegister::DaifSet, 0), + (0xd5034fdf, SystemRegister::DaifSet, 15), + (0xd50342ff, SystemRegister::DaifClear, 2), + ] { + assert_eq!( + decode(word), + Ok(Instruction::System(System { + read: false, + register, + rt: 31, + immediate + })), + "{word:#010x}" + ); + } + } + #[test] fn assembler_hints_barriers_and_traps() { use Instruction::*; @@ -2749,9 +3157,6 @@ mod tests { } assert_eq!(decode(0xd50b7423), Ok(DcZva(3))); assert_eq!(decode(0xd503309f), Ok(Dsb(0))); - assert!(!Dsb(0).terminates()); - assert!(Isb(0).terminates()); - assert!(Eret.terminates()); } #[test] fn prefetch_and_system_operations_keep_their_operands() { @@ -2769,7 +3174,7 @@ mod tests { Ok(Prefetch(super::Prefetch { op: 0, rn: 2, - addressing: Addressing::Offset(-8), + addressing: Addressing::Unscaled(-8), })) ); assert_eq!( @@ -2780,7 +3185,7 @@ mod tests { addressing: Addressing::Register { rm: 4, extend: Extend::Uxtx, - amount: 3, + amount: Some(3), }, })) ); @@ -2890,9 +3295,9 @@ mod tests { 0x6f000422, 0xffff_fffe_ffff_fffe, 0xffff_fffe_ffff_fffe, - Set, + SetInverted, ), - (0x2f006489, 0xfbff_ffff_fbff_ffff, 0, Set), + (0x2f006489, 0xfbff_ffff_fbff_ffff, 0, SetInverted), ( 0x4f00a468, 0x0300_0300_0300_0300, @@ -2954,6 +3359,58 @@ mod tests { } } #[test] + fn simd_modified_immediates_keep_their_encoding_class() { + use ImmForm::*; + // (word, form, imm8, q); every word checked against GNU `objdump`. + for (word, form, imm8, q) in [ + (0x4f000400, Word { shift: 0 }, 0, true), + (0x0f000400, Word { shift: 0 }, 0, false), + (0x6f00e401, Double, 0, true), + (0x2f00e400, Double, 0, false), + (0x4f00e400, Byte, 0, true), + (0x0f00e460, Byte, 3, false), + (0x6f0707ff, Word { shift: 0 }, 0xff, true), + (0x6f00c400, WordMask { shift: 8 }, 0, true), + (0x4f00d420, WordMask { shift: 16 }, 1, true), + (0x0f006400, Word { shift: 24 }, 0, false), + (0x4f00a420, Half { shift: 8 }, 1, true), + (0x0f008420, Half { shift: 0 }, 1, false), + (0x4f00f400, FmovSingle, 0, true), + (0x6f00f400, FmovDouble, 0, true), + (0x4f00fc00, FmovHalf, 0, true), + (0x0f00fc00, FmovHalf, 0, false), + ] { + let Ok(Instruction::SimdImm(simd)) = decode(word) else { + panic!("{word:08x} is not a simd immediate") + }; + assert_eq!( + (simd.form, simd.imm8, simd.q), + (form, imm8, q), + "{word:08x}" + ); + } + // `mvni` and `movi` with the same pattern differ in `combine`. + let Ok(Instruction::SimdImm(mvni)) = decode(0x6f00045f) else { + panic!() + }; + assert_eq!(mvni.combine, ImmCombine::SetInverted); + // `fmov v.4h, #2.0` expands to the half-precision pattern in every lane. + let Ok(Instruction::SimdImm(half)) = decode(0x0f00fc00) else { + panic!() + }; + assert_eq!((half.low, half.high), (0x4000_4000_4000_4000, 0)); + // Unallocated in the modified-immediate class (`objdump`: `.inst`, undefined): + // bit 11 set outside the half-precision `fmov`, `fmov .2d` without `Q`, and + // `op` set with the half-precision form. + for word in [0x0f000ced, 0x4f00af48, 0x0f009c28, 0x2f00f420, 0x6f00fc20] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:08x}" + ); + } + } + #[test] fn signed_branches_and_address_generation_preserve_negative_offsets() { assert_eq!(decode(0x17ffffff), Ok(Instruction::B(-4))); assert_eq!( @@ -3014,6 +3471,57 @@ mod tests { assert_eq!(ext.rn, 29); assert_eq!(ext.rm, 0); assert_eq!(ext.imm, 8); + assert!(ext.q); + // `ext v0.8b, v1.8b, v2.8b, #7` has Q clear; with an index of 8 or more + // (`objdump`: undefined) there is no `.8b` form. + let Ok(Instruction::SimdExt(ext)) = decode(0x2e023820) else { + panic!("not ext") + }; + assert_eq!((ext.imm, ext.q), (7, false)); + assert_eq!(decode(0x2e164372), Err(DecodeError::UnsupportedInstruction)); + } + /// `objdump` calls every one of these `.inst ... ; undefined` or decodes them as a + /// different instruction (half-precision `fmla`/`fmulx`/`frecps`/`fmaxnm`, `luti2`, + /// `luti4`, indexed multiply); none may decode as the vector move, `tbl`, ALU or + /// immediate instruction whose mask they once satisfied. + #[test] + fn unallocated_or_other_simd_encodings_are_rejected() { + for word in [ + 0x4e5a0d5f, // fmla v31.8h (was `dup`) + 0x0e471c94, // fmulx v20.4h (was `mov v.b[n], w`) + 0x0e5b3cd8, // frecps v24.4h (was `umov`) + 0x4e4105db, // fmaxnm v27.8h (was `dup` element) + 0x4e8b21d7, // undefined table encoding + 0x4e86516b, // luti2 + 0x4e5c136f, // luti4 + 0x6eee6d74, // umin .2d + 0x4efb2521, // shsub .2d + 0x4ee6a4d4, // smaxp .2d + 0x4ef99d5a, // mul .2d + 0x0e1c1fba, // ins (general) with Q clear + 0x4e013c20, // umov w, v.b with Q set + 0x0e083c20, // umov x, v.d with Q clear + 0x4fab1d6c, // indexed-element class, bit 24 set + 0x2e5a0d5f, // fp16 class under U=1 + 0x6e000420, // dup/ins element with imm5 = 0 + ] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:08x}" + ); + } + } + #[test] + fn decodes_bitwise_three_same_with_the_64_bit_form_selected_by_size() { + // `bif v0.8b, v31.8b, v30.8b` and `orn v0.8b, v1.8b, v0.8b`: `size` is part of + // the opcode, so `size == 3` with `Q` clear is valid here. + for (word, op) in [(0x2efe1fe0, SimdAluOp::Bif), (0x0ee01c20, SimdAluOp::Orn)] { + let Ok(Instruction::SimdAlu(alu)) = decode(word) else { + panic!("{word:08x}") + }; + assert_eq!((alu.op, alu.size, alu.q), (op, 3, false), "{word:08x}"); + } } #[test] fn decodes_single_structure_simd_memory() { @@ -3025,6 +3533,7 @@ mod tests { rn: 1, rt: 0, addressing: Addressing::Offset(0), + access: SimdAccess::Ld1 { esize: 1 }, })) ); assert_eq!( @@ -3035,6 +3544,7 @@ mod tests { rn: 1, rt: 0, addressing: Addressing::PostIndex(16), + access: SimdAccess::Ld1 { esize: 1 }, })) ); } @@ -3061,6 +3571,7 @@ mod tests { rn, rt, rt2, + order: MemoryOrder::default(), })) }; // ldxp x9, x21, [x8]; stxp w11, x19, x10, [x8]; ldxp w0, wzr, [x0] @@ -3355,7 +3866,8 @@ mod tests { rd: 2, rn: 2, to_fp: false, - double: true + double: true, + upper: false, })) ); assert_eq!( @@ -3364,7 +3876,8 @@ mod tests { rd: 2, rn: 2, to_fp: true, - double: true + double: true, + upper: false, })) ); assert_eq!( @@ -3373,7 +3886,8 @@ mod tests { rd: 2, rn: 1, to_fp: false, - double: false + double: false, + upper: false, })) ); assert_eq!( @@ -3382,7 +3896,29 @@ mod tests { rd: 1, rn: 2, to_fp: true, - double: false + double: false, + upper: false, + })) + ); + // `fmov v0.d[1], x3` / `fmov x3, v0.d[1]` move the upper doubleword. + assert_eq!( + decode(0x9eaf0060), + Ok(Instruction::SimdFmov(SimdFmov { + rd: 0, + rn: 3, + to_fp: true, + double: true, + upper: true, + })) + ); + assert_eq!( + decode(0x9eae0003), + Ok(Instruction::SimdFmov(SimdFmov { + rd: 3, + rn: 0, + to_fp: false, + double: true, + upper: true, })) ); } @@ -3493,4 +4029,426 @@ mod tests { } } } + fn memory(op: MemoryOp, size: Size, rt: u8, rn: u8, addressing: Addressing) -> Memory { + Memory { + op, + size, + rn, + rt, + addressing, + signed: SignExtend::None, + ordered: None, + } + } + #[test] + fn ordered_memory_accesses_keep_their_class() { + use MemoryOp::{Load, Store}; + let ordered = |op, size, rt, rn, ordered| { + Ok(Instruction::Memory(Memory { + ordered: Some(ordered), + ..memory(op, size, rt, rn, Addressing::Offset(0)) + })) + }; + // ldar w0, [x1]; ldar x0, [sp]; ldarb w0, [x1]; stlr x0, [sp]; stlrh w0, [x1] + assert_eq!( + decode(0x88dffc20), + ordered(Load, Size::Word, 0, 1, Ordered::Acquire) + ); + assert_eq!( + decode(0xc8dfffe0), + ordered(Load, Size::Double, 0, 31, Ordered::Acquire) + ); + assert_eq!( + decode(0x08dffc20), + ordered(Load, Size::Byte, 0, 1, Ordered::Acquire) + ); + assert_eq!( + decode(0xc89fffe0), + ordered(Store, Size::Double, 0, 31, Ordered::Release) + ); + assert_eq!( + decode(0x489ffc20), + ordered(Store, Size::Half, 0, 1, Ordered::Release) + ); + // ldlar x0, [x1]; stllrb w0, [x1] + assert_eq!( + decode(0xc8df7c20), + ordered(Load, Size::Double, 0, 1, Ordered::LimitedAcquire) + ); + assert_eq!( + decode(0x089f7c20), + ordered(Store, Size::Byte, 0, 1, Ordered::LimitedRelease) + ); + // ldapr w0, [x1]; ldaprh w0, [x1] + assert_eq!( + decode(0xb8bfc020), + ordered(Load, Size::Word, 0, 1, Ordered::AcquirePc) + ); + assert_eq!( + decode(0x78bfc020), + ordered(Load, Size::Half, 0, 1, Ordered::AcquirePc) + ); + // The unscaled RCpc forms: ldapur w0, [x1, #4]; stlur w0, [x1, #4]; + // ldapursb x0, [x1, #4]. + assert_eq!( + decode(0x99404020), + Ok(Instruction::Memory(Memory { + ordered: Some(Ordered::AcquirePc), + ..memory(Load, Size::Word, 0, 1, Addressing::Unscaled(4)) + })) + ); + assert_eq!( + decode(0x99004020), + Ok(Instruction::Memory(Memory { + ordered: Some(Ordered::Release), + ..memory(Store, Size::Word, 0, 1, Addressing::Unscaled(4)) + })) + ); + assert_eq!( + decode(0x19804020), + Ok(Instruction::Memory(Memory { + signed: SignExtend::To64, + ordered: Some(Ordered::AcquirePc), + ..memory(Load, Size::Byte, 0, 1, Addressing::Unscaled(4)) + })) + ); + } + #[test] + fn exclusives_and_atomics_keep_acquire_and_release() { + let exclusive = |op, size, rs, rt, acquire, release| { + Ok(Instruction::Exclusive(Exclusive { + op, + size, + rn: 1, + rt, + rs, + order: MemoryOrder { acquire, release }, + })) + }; + // ldxr w0, [x1]; ldaxr w0, [x1]; stxr w2, w0, [x1]; stlxr w2, w0, [x1] + assert_eq!( + decode(0x885f7c20), + exclusive(MemoryOp::Load, Size::Word, 31, 0, false, false) + ); + assert_eq!( + decode(0x885ffc20), + exclusive(MemoryOp::Load, Size::Word, 31, 0, true, false) + ); + assert_eq!( + decode(0x88027c20), + exclusive(MemoryOp::Store, Size::Word, 2, 0, false, false) + ); + assert_eq!( + decode(0x8802fc20), + exclusive(MemoryOp::Store, Size::Word, 2, 0, false, true) + ); + let atomic = |kind, size, rs, rt, rt2, acquire, release| { + Ok(Instruction::Atomic(Atomic { + kind, + size, + rs, + rn: 2, + rt, + rt2, + order: MemoryOrder { acquire, release }, + })) + }; + // cas w0, w1, [x2]; casa; casl; casal x0, x1, [x2] + for (word, size, acquire, release) in [ + (0x88a07c41, Size::Word, false, false), + (0x88e07c41, Size::Word, true, false), + (0x88a0fc41, Size::Word, false, true), + (0xc8e0fc41, Size::Double, true, true), + ] { + assert_eq!( + decode(word), + atomic(AtomicKind::Cas, size, 0, 1, 0, acquire, release), + "{word:#010x}" + ); + } + // casp, caspa, caspl, caspal x0, x1, x2, x3, [x4] share one kind. + let casp = |word, acquire, release| { + assert_eq!( + decode(word), + Ok(Instruction::Atomic(Atomic { + kind: AtomicKind::Casp, + size: Size::Double, + rs: 0, + rn: 4, + rt: 2, + rt2: 0, + order: MemoryOrder { acquire, release }, + })), + "{word:#010x}" + ); + }; + casp(0x48207c82, false, false); + casp(0x48607c82, true, false); + casp(0x4820fc82, false, true); + casp(0x4860fc82, true, true); + // swpal x0, xzr, [x2]; ldadda w0, wzr, [x1] and ldaddl w0, w1, [x2] + assert_eq!( + decode(0xf8e0805f), + atomic(AtomicKind::Swap, Size::Double, 0, 31, 0, true, true) + ); + assert_eq!( + decode(0xb8a0003f), + Ok(Instruction::Atomic(Atomic { + kind: AtomicKind::Add, + size: Size::Word, + rs: 0, + rn: 1, + rt: 31, + rt2: 0, + order: MemoryOrder { + acquire: true, + release: false + }, + })) + ); + assert_eq!( + decode(0xb8600041), + atomic(AtomicKind::Add, Size::Word, 0, 1, 0, false, true) + ); + // ldaxp w0, w1, [x2]; stlxp w3, x0, x1, [x2] + assert_eq!( + decode(0x887f8440), + atomic(AtomicKind::LoadPair, Size::Word, 31, 0, 1, true, false) + ); + assert_eq!( + decode(0xc8238440), + atomic(AtomicKind::StorePair, Size::Double, 3, 0, 1, false, true) + ); + } + #[test] + fn offset_forms_keep_their_encoding() { + use MemoryOp::Load; + // ldr x0, [x1, #8]; ldur x0, [x1, #8]; ldtr w0, [x1, #4]; ldur w0, [x1] + assert_eq!( + decode(0xf9400420), + Ok(Instruction::Memory(memory( + Load, + Size::Double, + 0, + 1, + Addressing::Offset(8) + ))) + ); + assert_eq!( + decode(0xf8408020), + Ok(Instruction::Memory(memory( + Load, + Size::Double, + 0, + 1, + Addressing::Unscaled(8) + ))) + ); + assert_eq!( + decode(0xb8404820), + Ok(Instruction::Memory(memory( + Load, + Size::Word, + 0, + 1, + Addressing::Unprivileged(4) + ))) + ); + assert_eq!( + decode(0xb8400020), + Ok(Instruction::Memory(memory( + Load, + Size::Word, + 0, + 1, + Addressing::Unscaled(0) + ))) + ); + // ldr q0, [x1]; ldur q0, [x1, #16]; ldur b0, [x1, #1]; ldr b0, [x1, #1] + let simd = |word, bytes, addressing| { + assert_eq!( + decode(word), + Ok(Instruction::SimdMemory(SimdMemory { + op: MemoryOp::Load, + bytes, + rn: 1, + rt: 0, + addressing, + access: SimdAccess::Register, + })), + "{word:#010x}" + ); + }; + simd(0x3dc00020, 16, Addressing::Offset(0)); + simd(0x3cc10020, 16, Addressing::Unscaled(16)); + simd(0x3c401020, 1, Addressing::Unscaled(1)); + simd(0x3d400420, 1, Addressing::Offset(1)); + // prfm pldl1keep, [x1, #8]; prfum pldl1keep, [x1, #8] + assert_eq!( + decode(0xf9800420), + Ok(Instruction::Prefetch(Prefetch { + op: 0, + rn: 1, + addressing: Addressing::Offset(8) + })) + ); + assert_eq!( + decode(0xf8808020), + Ok(Instruction::Prefetch(Prefetch { + op: 0, + rn: 1, + addressing: Addressing::Unscaled(8), + })) + ); + } + #[test] + fn register_offsets_keep_whether_they_are_shifted() { + let register = |word, extend, amount| { + assert_eq!( + decode(word), + Ok(Instruction::Memory(memory( + MemoryOp::Load, + Size::Byte, + 0, + 1, + Addressing::Register { + rm: 2, + extend, + amount + } + ))), + "{word:#010x}" + ); + }; + // ldrb w0, [x1, x2, lsl #0]; ldrb w0, [x1, x2]; ldrb w0, [x1, w2, sxtw #0]; + // ldrb w0, [x1, w2, sxtw] + register(0x38627820, Extend::Uxtx, Some(0)); + register(0x38626820, Extend::Uxtx, None); + register(0x3862d820, Extend::Sxtw, Some(0)); + register(0x3862c820, Extend::Sxtw, None); + // str b2, [x11, x0, lsl #0]; str q2, [x11, x0, lsl #4]; str q2, [x11, x0] + let simd = |word, bytes, amount| { + assert_eq!( + decode(word), + Ok(Instruction::SimdMemory(SimdMemory { + op: MemoryOp::Store, + bytes, + rn: 11, + rt: 2, + addressing: Addressing::Register { + rm: 0, + extend: Extend::Uxtx, + amount + }, + access: SimdAccess::Register, + })), + "{word:#010x}" + ); + }; + simd(0x3c207962, 1, Some(0)); + simd(0x3ca07962, 16, Some(4)); + simd(0x3ca06962, 16, None); + } + #[test] + fn non_temporal_pairs_keep_their_identity() { + let Ok(Instruction::Pair(pair)) = decode(0xa8400440) else { + panic!("ldnp x0, x1, [x2]"); + }; + assert_eq!( + (pair.op, pair.size, pair.addressing, pair.non_temporal), + (MemoryOp::Load, Size::Double, Addressing::Offset(0), true) + ); + let Ok(Instruction::Pair(pair)) = decode(0x28010440) else { + panic!("stnp w0, w1, [x2, #8]"); + }; + assert_eq!( + (pair.op, pair.size, pair.addressing, pair.non_temporal), + (MemoryOp::Store, Size::Word, Addressing::Offset(8), true) + ); + let Ok(Instruction::Pair(pair)) = decode(0xa9400440) else { + panic!("ldp x0, x1, [x2]"); + }; + assert!(!pair.non_temporal); + let Ok(Instruction::SimdPair(pair)) = decode(0xac3f0440) else { + panic!("stnp q0, q1, [x2, #-32]"); + }; + assert_eq!( + (pair.op, pair.bytes, pair.addressing, pair.non_temporal), + (MemoryOp::Store, 16, Addressing::Offset(-32), true) + ); + let Ok(Instruction::SimdPair(pair)) = decode(0xadc08440) else { + panic!("ldp q0, q1, [x2, #16]!"); + }; + assert!(!pair.non_temporal); + } + #[test] + fn one_register_ld1_keeps_its_arrangement() { + let ld1 = |word, op, bytes, addressing, esize| { + assert_eq!( + decode(word), + Ok(Instruction::SimdMemory(SimdMemory { + op, + bytes, + rn: 1, + rt: 0, + addressing, + access: SimdAccess::Ld1 { esize }, + })), + "{word:#010x}" + ); + }; + // ld1 {v0.8b}, [x1]; ld1 {v0.8h}, [x1]; ld1 {v0.2s}, [x1]; ld1 {v0.1d}, [x1] + ld1(0x0c407020, MemoryOp::Load, 8, Addressing::Offset(0), 1); + ld1(0x4c407420, MemoryOp::Load, 16, Addressing::Offset(0), 2); + ld1(0x0c407820, MemoryOp::Load, 8, Addressing::Offset(0), 4); + ld1(0x0c407c20, MemoryOp::Load, 8, Addressing::Offset(0), 8); + // ld1 {v0.4s}, [x1], #16; st1 {v0.2s}, [x1], #8 + ld1(0x4cdf7820, MemoryOp::Load, 16, Addressing::PostIndex(16), 4); + ld1(0x0c9f7820, MemoryOp::Store, 8, Addressing::PostIndex(8), 4); + // A register post-index is a structure access. + assert!(matches!(decode(0x4cc27020), Ok(Instruction::SimdStruct(_)))); + } + #[test] + fn range_prefetch_and_ldrsw_literal_decode() { + // rprfm pldkeep, x2, [x1]; rprfm pldstrm, xzr, [sp]; rprfm #2, x2, [x1] + for (word, op, rm, rn) in [ + (0xf8a24838, 0, 2, 1), + (0xf8bf4bfc, 4, 31, 31), + (0xf8a2483a, 2, 2, 1), + ] { + assert_eq!( + decode(word), + Ok(Instruction::RangePrefetch(RangePrefetch { op, rm, rn })), + "{word:#010x}" + ); + } + // ldrsw x0, #8 + assert_eq!( + decode(0x98000040), + Ok(Instruction::Literal(Literal { + size: Size::Word, + rt: 0, + offset: 8, + signed: SignExtend::To64, + })) + ); + } + #[test] + fn unallocated_memory_words_are_rejected() { + // Each is `.inst ... ; undefined` for GNU objdump 2.46: PRFM (register) with a + // byte or halfword extend, LDAPR with the wrong A/R bits or a non-zero Rs, an + // unprivileged SIMD&FP access, LDNP with the `ldpsw` opc, and LDAPURS* with an + // impossible size. + for word in [ + 0xf8a20838, 0xf8a21838, 0xf8a28838, 0xf8a29838, 0xf8a2a838, 0xf8a20837, 0xb8ffc020, + 0xb87fc020, 0xb83fc020, 0x78ffc020, 0xb8bdc020, 0x3c400820, 0x3c000820, 0x68400440, + 0x68000440, 0x99c04020, 0xd9804020, + ] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:#010x}" + ); + } + } } diff --git a/src/isa/aarch64/src/flow.rs b/src/isa/aarch64/src/flow.rs new file mode 100644 index 0000000..3cfc298 --- /dev/null +++ b/src/isa/aarch64/src/flow.rs @@ -0,0 +1,137 @@ +//! Control-flow classification of A64 instructions, for analysis. +//! +//! [`Instruction::flow`] answers the CFG question: where can execution go +//! next. It is architectural. Whether a particular translator cuts its blocks +//! at loads, stores or system instructions is that translator's policy and is +//! not decided here. +use crate::decode::{IndirectKind, Instruction, decode, is_undefined}; + +/// Where execution can go after an instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Flow { + /// Continues at `pc + 4`. + Next, + /// Unconditional direct branch. + Jump(u64), + /// Conditional direct branch: taken target, otherwise `pc + 4`. + CondJump(u64), + /// Direct call (`bl`); returns to `pc + 4`. + Call(u64), + /// Indirect call (`blr`); returns to `pc + 4`. + CallIndirect, + /// Indirect jump (`br`); no known successor. + JumpIndirect, + /// `ret`, `eret`. + Return, + /// Does not continue (`brk`, permanently undefined words). + Stop, + /// The word is not an instruction this crate decodes, so where it goes is + /// unknown. Analysis that must keep walking treats it as falling through + /// (see [`Flow::falls_through`]); it never ends a block. + Unrecognized, +} + +impl Flow { + /// Whether the instruction after this one (`pc + 4`) is a successor. + pub fn falls_through(self) -> bool { + matches!( + self, + Self::Next + | Self::CondJump(_) + | Self::Call(_) + | Self::CallIndirect + | Self::Unrecognized + ) + } + /// The statically known branch or call target. + pub fn target(self) -> Option { + match self { + Self::Jump(t) | Self::CondJump(t) | Self::Call(t) => Some(t), + _ => None, + } + } + /// Whether this instruction ends a basic block. + pub fn ends_block(self) -> bool { + !matches!(self, Self::Next | Self::Unrecognized) + } +} + +impl Instruction { + /// Where execution can go after this instruction when it is located at `pc`. + pub fn flow(&self, pc: u64) -> Flow { + let rel = |off: i64| pc.wrapping_add(off as u64); + match self { + Self::B(o) => Flow::Jump(rel(*o)), + Self::BCond(b) => Flow::CondJump(rel(b.offset)), + Self::TestBranch(t) => Flow::CondJump(rel(t.offset)), + Self::Call(c) if c.link => Flow::Call(rel(c.target)), + Self::Call(c) => Flow::Jump(rel(c.target)), + Self::Indirect(i) => match i.kind { + IndirectKind::Jump => Flow::JumpIndirect, + IndirectKind::Call => Flow::CallIndirect, + IndirectKind::Return => Flow::Return, + }, + Self::Eret => Flow::Return, + Self::Brk(_) => Flow::Stop, + _ => Flow::Next, + } + } +} + +/// Classify the instruction `word` located at `pc`. Permanently undefined +/// words are [`Flow::Stop`]; words the decoder does not recognise are +/// [`Flow::Unrecognized`]. +pub fn flow(word: u32, pc: u64) -> Flow { + match decode(word) { + Ok(instruction) => instruction.flow(pc), + Err(_) if is_undefined(word) => Flow::Stop, + Err(_) => Flow::Unrecognized, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies() { + assert_eq!(flow(0xd65f_03c0, 0x100), Flow::Return); + assert_eq!(flow(0xd61f_0200, 0x100), Flow::JumpIndirect); // br x16 + assert_eq!(flow(0xd63f_0100, 0x100), Flow::CallIndirect); // blr x8 + assert_eq!(flow(0x1400_0004, 0x100), Flow::Jump(0x110)); // b .+16 + assert_eq!(flow(0x97ff_ffff, 0x100), Flow::Call(0xfc)); // bl .-4 + assert_eq!(flow(0x3400_00e8, 0x100), Flow::CondJump(0x11c)); // cbz w8, .+28 + assert_eq!(flow(0xd503_201f, 0x100), Flow::Next); // nop + assert_eq!(flow(0xd420_0020, 0x100), Flow::Stop); // brk #1 + assert_eq!(flow(0xd400_0021, 0x100), Flow::Next); // svc #1 + assert_eq!(flow(0xe7ff_deff, 0x100), Flow::Stop); // udf: permanently undefined + } + + #[test] + fn ret_and_br_stay_distinct_in_the_decoded_instruction() { + use crate::decode::Indirect; + let ret = decode(0xd65f_03c0).unwrap(); + let br = decode(0xd61f_0200).unwrap(); + assert_eq!( + ret, + Instruction::Indirect(Indirect { + rn: 30, + kind: IndirectKind::Return + }) + ); + assert_eq!( + br, + Instruction::Indirect(Indirect { + rn: 16, + kind: IndirectKind::Jump + }) + ); + } + + #[test] + fn unrecognized_words_neither_end_a_block_nor_stop_a_walk() { + assert!(Flow::Unrecognized.falls_through()); + assert!(!Flow::Unrecognized.ends_block()); + assert!(!Flow::Stop.falls_through()); + } +} diff --git a/src/isa/aarch64/src/format/fp.rs b/src/isa/aarch64/src/format/fp.rs new file mode 100644 index 0000000..8176276 --- /dev/null +++ b/src/isa/aarch64/src/format/fp.rs @@ -0,0 +1,357 @@ +//! Scalar floating point and the general-register / SIMD&FP register `fmov`. +//! +//! Owns `Fp` and `SimdFmov`. + +use super::Context; +use super::reg::{arrangement, fp_scalar, gpr}; +use crate::decode::{Instruction, SimdFmov, Width}; +use crate::fp::{Binary, Fused, Op, Precision, Unary}; +use core::fmt::{self, Formatter}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + let _ = cx; + match instruction { + Instruction::Fp(op) => write_op(f, *op), + Instruction::SimdFmov(op) => write_fmov(f, *op), + other => unreachable!("{other:?} is not a floating-point instruction"), + } +} + +/// Size in bytes of the scalar register for a precision. +fn size(p: Precision) -> u8 { + match p { + Precision::Single => 4, + Precision::Double => 8, + } +} + +fn width(wide: bool) -> Width { + if wide { Width::X64 } else { Width::W32 } +} + +/// The condition field as binutils spells it; the field is four bits. +fn condition(f: &mut Formatter<'_>, cond: u32) -> fmt::Result { + const NAMES: [&str; 16] = [ + "eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc", "hi", "ls", "ge", "lt", "gt", "le", "al", + "nv", + ]; + f.write_str(NAMES[(cond & 15) as usize]) +} + +fn write_fmov(f: &mut Formatter<'_>, op: SimdFmov) -> fmt::Result { + f.write_str("fmov ")?; + // The upper doubleword is named as the lane `Vn.D[1]`. + let vector = |f: &mut Formatter<'_>, r: u8| -> fmt::Result { + if op.upper { + write!(f, "v{r}.d[1]") + } else { + fp_scalar(f, if op.double { 8 } else { 4 }, r) + } + }; + if op.to_fp { + vector(f, op.rd)?; + f.write_str(", ")?; + gpr(f, width(op.double), op.rn) + } else { + gpr(f, width(op.double), op.rd)?; + f.write_str(", ")?; + vector(f, op.rn) + } +} + +/// `mnemonic rd, rn` over scalar registers of precisions `pd` and `pn`. +fn two( + f: &mut Formatter<'_>, + mnemonic: &str, + (pd, rd): (Precision, u8), + (pn, rn): (Precision, u8), +) -> fmt::Result { + write!(f, "{mnemonic} ")?; + fp_scalar(f, size(pd), rd)?; + f.write_str(", ")?; + fp_scalar(f, size(pn), rn) +} + +fn regs(f: &mut Formatter<'_>, p: Precision, list: &[u8]) -> fmt::Result { + for (i, &r) in list.iter().enumerate() { + if i != 0 { + f.write_str(", ")?; + } + fp_scalar(f, size(p), r)?; + } + Ok(()) +} + +fn write_op(f: &mut Formatter<'_>, op: Op) -> fmt::Result { + match op { + Op::Unary { kind, p, rd, rn } => { + let mnemonic = match kind { + Unary::Move => "fmov", + Unary::Abs => "fabs", + Unary::Neg => "fneg", + Unary::Sqrt => "fsqrt", + }; + two(f, mnemonic, (p, rd), (p, rn)) + } + Op::Convert { p, rd, rn } => { + let to = match p { + Precision::Single => Precision::Double, + Precision::Double => Precision::Single, + }; + two(f, "fcvt", (to, rd), (p, rn)) + } + Op::Binary { + kind, + p, + rd, + rn, + rm, + } => { + let mnemonic = match kind { + Binary::Mul => "fmul", + Binary::Div => "fdiv", + Binary::Add => "fadd", + Binary::Sub => "fsub", + Binary::Max => "fmax", + Binary::Min => "fmin", + Binary::MaxNum => "fmaxnm", + Binary::MinNum => "fminnm", + Binary::Nmul => "fnmul", + }; + write!(f, "{mnemonic} ")?; + regs(f, p, &[rd, rn, rm]) + } + Op::Fused { + kind, + p, + rd, + rn, + rm, + ra, + } => { + let mnemonic = match kind { + Fused::Madd => "fmadd", + Fused::Msub => "fmsub", + Fused::Nmadd => "fnmadd", + Fused::Nmsub => "fnmsub", + }; + write!(f, "{mnemonic} ")?; + regs(f, p, &[rd, rn, rm, ra]) + } + Op::Compare { + p, + rn, + rm, + signaling, + } => { + f.write_str(if signaling { "fcmpe " } else { "fcmp " })?; + fp_scalar(f, size(p), rn)?; + f.write_str(", ")?; + match rm { + Some(rm) => fp_scalar(f, size(p), rm), + None => f.write_str("#0.0"), + } + } + Op::CondCompare { + p, + rn, + rm, + cond, + nzcv, + } => { + f.write_str("fccmp ")?; + regs(f, p, &[rn, rm])?; + write!(f, ", #{:#x}, ", nzcv & 15)?; + condition(f, cond) + } + Op::Select { + p, + rd, + rn, + rm, + cond, + } => { + f.write_str("fcsel ")?; + regs(f, p, &[rd, rn, rm])?; + f.write_str(", ")?; + condition(f, cond) + } + Op::Immediate { p, rd, imm8 } => { + f.write_str("fmov ")?; + fp_scalar(f, size(p), rd)?; + f.write_str(", #")?; + expanded_immediate(f, imm8 as u8) + } + Op::IntToFloat { + signed, + p, + wide, + rd, + rn, + } => { + f.write_str(if signed { "scvtf " } else { "ucvtf " })?; + fp_scalar(f, size(p), rd)?; + f.write_str(", ")?; + gpr(f, width(wide), rn) + } + Op::FloatToInt { + signed, + p, + wide, + rd, + rn, + } => { + f.write_str(if signed { "fcvtzs " } else { "fcvtzu " })?; + gpr(f, width(wide), rd)?; + f.write_str(", ")?; + fp_scalar(f, size(p), rn) + } + Op::ScalarIntToFloat { signed, p, rd, rn } => { + two(f, if signed { "scvtf" } else { "ucvtf" }, (p, rd), (p, rn)) + } + Op::ScalarFloatToInt { signed, p, rd, rn } => two( + f, + if signed { "fcvtzs" } else { "fcvtzu" }, + (p, rd), + (p, rn), + ), + Op::VectorSign { + negate, + double, + q, + rd, + rn, + } => { + f.write_str(if negate { "fneg " } else { "fabs " })?; + let esize = if double { 3 } else { 2 }; + write!(f, "v{rd}.")?; + arrangement(f, esize, q)?; + write!(f, ", v{rn}.")?; + arrangement(f, esize, q) + } + } +} + +/// Print the VFPExpandImm value of `imm8` the way binutils does: `%.18e`. +/// +/// With `imm8 = a:bcd:efgh` the value is `±(16 + efgh) / 16 * 2^exp` where `exp` is +/// the signed three-bit `NOT(b):c:d` plus one (`-3..=4`). That is `n / 2^j` with +/// `n` in `16..=31` and `j` in `0..=7`, so `n * 5^j / 10^j` is an exact decimal +/// integer of at most seven digits. +fn expanded_immediate(f: &mut Formatter<'_>, imm8: u8) -> fmt::Result { + let negative = imm8 & 0x80 != 0; + let e = i32::from((imm8 >> 4) & 7); + let exp = if e < 4 { e + 1 } else { e - 7 }; + let n = 16 + u32::from(imm8 & 15); + let j = (4 - exp) as u32; + // `j` is `0..=7`; `n * 5^j` is at most 31 * 78125. + let digits_value = n * 5u32.pow(j); + let mut digits = [0u8; 10]; + let mut count = 0; + let mut rest = digits_value; + while rest != 0 { + digits[count] = (rest % 10) as u8; + rest /= 10; + count += 1; + } + // `digits[count - 1]` is the most significant digit. + let decimal_exponent = count as i32 - 1 - j as i32; + if negative { + f.write_str("-")?; + } + write!(f, "{}.", digits[count - 1])?; + for i in 1..19 { + let d = if i < count { digits[count - 1 - i] } else { 0 }; + write!(f, "{d}")?; + } + let sign = if decimal_exponent < 0 { '-' } else { '+' }; + write!(f, "e{sign}{:02}", decimal_exponent.unsigned_abs()) +} + +#[cfg(test)] +mod tests { + use crate::format::word; + use core::fmt::Write; + + struct Buf { + bytes: [u8; 96], + len: usize, + } + + impl Write for Buf { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + self.bytes[self.len..self.len + s.len()].copy_from_slice(s.as_bytes()); + self.len += s.len(); + Ok(()) + } + } + + fn check(w: u32, expected: &str) { + let mut buf = Buf { + bytes: [0; 96], + len: 0, + }; + write!(buf, "{}", word(w)).unwrap(); + assert_eq!( + core::str::from_utf8(&buf.bytes[..buf.len]).unwrap(), + expected, + "{w:#010x}" + ); + } + + #[test] + fn scalar_floating_point() { + check(0x1e602008, "fcmp d0, #0.0"); + check(0x1e222020, "fcmp s1, s2"); + check(0x1e222030, "fcmpe s1, s2"); + check(0x1e202038, "fcmpe s1, #0.0"); + check(0x1e7f2010, "fcmpe d0, d31"); + check(0x1e221425, "fccmp s1, s2, #0x5, ne"); + check(0x1e60e7ef, "fccmp d31, d0, #0xf, al"); + check(0x1e633c41, "fcsel d1, d2, d3, cc"); + check(0x1e22c020, "fcvt d0, s1"); + check(0x1e6243e0, "fcvt s0, d31"); + check(0x1e638841, "fnmul d1, d2, d3"); + check(0x1f239041, "fnmsub s1, s2, s3, s4"); + check(0x1e204041, "fmov s1, s2"); + check(0x1e21c041, "fsqrt s1, s2"); + check(0x1e636841, "fmaxnm d1, d2, d3"); + } + + #[test] + fn fmov_to_and_from_the_upper_doubleword() { + check(0x9eaf0060, "fmov v0.d[1], x3"); + check(0x9eae0003, "fmov x3, v0.d[1]"); + check(0x9e670042, "fmov d2, x2"); + } + + #[test] + fn immediates() { + check(0x1e78101f, "fmov d31, #-1.250000000000000000e-01"); + check(0x1e27f000, "fmov s0, #3.100000000000000000e+01"); + check(0x1e281000, "fmov s0, #1.250000000000000000e-01"); + } + + #[test] + fn conversions() { + check(0x1e620041, "scvtf d1, w2"); + check(0x9e230041, "ucvtf s1, x2"); + check(0x1e780020, "fcvtzs w0, d1"); + check(0x9e390020, "fcvtzu x0, s1"); + check(0x5e21d841, "scvtf s1, s2"); + check(0x7e61d841, "ucvtf d1, d2"); + check(0x5ee1b841, "fcvtzs d1, d2"); + check(0x7ea1b841, "fcvtzu s1, s2"); + check(0x0ea0f841, "fabs v1.2s, v2.2s"); + check(0x6ea0f841, "fneg v1.4s, v2.4s"); + check(0x4ee0f841, "fabs v1.2d, v2.2d"); + check(0x9e670020, "fmov d0, x1"); + check(0x9e660020, "fmov x0, d1"); + check(0x1e270020, "fmov s0, w1"); + check(0x1e260020, "fmov w0, s1"); + } +} diff --git a/src/isa/aarch64/src/format/integer.rs b/src/isa/aarch64/src/format/integer.rs new file mode 100644 index 0000000..502a758 --- /dev/null +++ b/src/isa/aarch64/src/format/integer.rs @@ -0,0 +1,877 @@ +//! Integer data processing: moves, add/subtract, logical, multiply, divide, shifts, +//! bitfields, conditional select/compare, `adr`/`adrp`. +//! +//! Owns `Movz`, `Movn`, `Movk`, `ArithImm`, `ArithReg`, `ArithExt`, `LogicReg`, +//! `LogicImm`, `Mul`, `MulLong`, `MulHigh`, `Adr`, `Csel`, `Variable`, `CondCompare`, +//! `ArithCarry`, `Unary`, `Bitfield` and `Extract`. + +use super::Context; +use super::reg::{condition, gpr, gpr_sp}; +use crate::decode::{ + AddCarry, Address, ArithOp, Arithmetic, ArithmeticExtended, ArithmeticReg, Bitfield, + BitfieldOp, CompareOperand, CondCompare, Condition, Extend, Extract, Instruction, Logic, + LogicImm, LogicOp, Multiply, MultiplyHigh, MultiplyLong, Select, Shift, Unary, UnaryOp, + Variable, VariableOp, Wide, Width, +}; +use core::fmt::{self, Formatter}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + match *instruction { + Instruction::Movz(w) => movz(f, w), + Instruction::Movn(w) => movn(f, w), + Instruction::Movk(w) => { + f.write_str("movk ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{:#x}", w.immediate)?; + lsl_suffix(f, w.shift) + } + Instruction::ArithImm(a) => arith_imm(f, a), + Instruction::ArithReg(a) => arith_reg(f, a), + Instruction::ArithExt(a) => arith_ext(f, a), + Instruction::LogicReg(l) => logic_reg(f, l), + Instruction::LogicImm(l) => logic_imm(f, l), + Instruction::Mul(m) => mul(f, m), + Instruction::MulLong(m) => mul_long(f, m), + Instruction::MulHigh(m) => mul_high(f, m), + Instruction::Adr(a) => adr(f, cx, a), + Instruction::Csel(s) => csel(f, s), + Instruction::Variable(v) => variable(f, v), + Instruction::CondCompare(c) => cond_compare(f, c), + Instruction::ArithCarry(a) => arith_carry(f, a), + Instruction::Unary(u) => unary(f, u), + Instruction::Bitfield(b) => bitfield(f, b), + Instruction::Extract(e) => extract(f, e), + _ => unreachable!("{instruction:?} is not an integer instruction"), + } +} + +/// `, lsl #n` unless `n` is zero. +fn lsl_suffix(f: &mut Formatter<'_>, amount: u8) -> fmt::Result { + if amount == 0 { + Ok(()) + } else { + write!(f, ", lsl #{amount}") + } +} + +fn shift_name(shift: Shift) -> &'static str { + match shift { + Shift::Lsl => "lsl", + Shift::Lsr => "lsr", + Shift::Asr => "asr", + Shift::Ror => "ror", + } +} + +fn extend_name(extend: Extend) -> &'static str { + match extend { + Extend::Uxtb => "uxtb", + Extend::Uxth => "uxth", + Extend::Uxtw => "uxtw", + Extend::Uxtx => "uxtx", + Extend::Sxtb => "sxtb", + Extend::Sxth => "sxth", + Extend::Sxtw => "sxtw", + Extend::Sxtx => "sxtx", + } +} + +/// `, #n`; a zero `lsl` is not printed (a zero `ror` is). +fn shift_operand(f: &mut Formatter<'_>, shift: Shift, amount: u8) -> fmt::Result { + if shift == Shift::Lsl && amount == 0 { + Ok(()) + } else { + write!(f, ", {} #{amount}", shift_name(shift)) + } +} + +fn invert(cond: Condition) -> Condition { + use Condition::*; + match cond { + Eq => Ne, + Ne => Eq, + Cs => Cc, + Cc => Cs, + Mi => Pl, + Pl => Mi, + Vs => Vc, + Vc => Vs, + Hi => Ls, + Ls => Hi, + Ge => Lt, + Lt => Ge, + Gt => Le, + Le => Gt, + Al => Nv, + Nv => Al, + } +} + +fn bits(width: Width) -> u16 { + u16::from(width as u8) +} + +/// The width's all-ones mask. +fn mask(width: Width) -> u64 { + match width { + Width::W32 => u64::from(u32::MAX), + Width::X64 => u64::MAX, + } +} + +fn movz(f: &mut Formatter<'_>, w: Wide) -> fmt::Result { + if w.immediate == 0 && w.shift != 0 { + f.write_str("movz ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{:#x}", w.immediate)?; + return lsl_suffix(f, w.shift); + } + let value = (u64::from(w.immediate) << (w.shift & 63)) & mask(w.width); + f.write_str("mov ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{value:#x}") +} + +fn movn(f: &mut Formatter<'_>, w: Wide) -> fmt::Result { + // `mov` only where a `movz` could not have produced the same value. + if (w.immediate == 0 && w.shift != 0) || (w.width == Width::W32 && w.immediate == 0xffff) { + f.write_str("movn ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{:#x}", w.immediate)?; + return lsl_suffix(f, w.shift); + } + let value = !(u64::from(w.immediate) << (w.shift & 63)) & mask(w.width); + f.write_str("mov ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{value:#x}") +} + +fn arith_imm(f: &mut Formatter<'_>, a: Arithmetic) -> fmt::Result { + let add = a.op == ArithOp::Add; + if !a.flags && add && a.immediate == 0 && !a.shifted && (a.rd == 31 || a.rn == 31) { + f.write_str("mov ")?; + gpr_sp(f, a.width, a.rd)?; + f.write_str(", ")?; + return gpr_sp(f, a.width, a.rn); + } + if a.flags && a.rd == 31 { + f.write_str(if add { "cmn " } else { "cmp " })?; + } else { + f.write_str(match (add, a.flags) { + (true, false) => "add ", + (true, true) => "adds ", + (false, false) => "sub ", + (false, true) => "subs ", + })?; + if a.flags { + gpr(f, a.width, a.rd)?; + } else { + gpr_sp(f, a.width, a.rd)?; + } + f.write_str(", ")?; + } + gpr_sp(f, a.width, a.rn)?; + write!(f, ", #{:#x}", a.immediate)?; + if a.shifted { + f.write_str(", lsl #12")?; + } + Ok(()) +} + +fn arith_reg(f: &mut Formatter<'_>, a: ArithmeticReg) -> fmt::Result { + let add = a.op == ArithOp::Add; + let mut rn = Some(a.rn); + if a.flags && a.rd == 31 { + f.write_str(if add { "cmn " } else { "cmp " })?; + } else if !add && a.rn == 31 { + f.write_str(if a.flags { "negs " } else { "neg " })?; + gpr(f, a.width, a.rd)?; + f.write_str(", ")?; + rn = None; + } else { + f.write_str(match (add, a.flags) { + (true, false) => "add ", + (true, true) => "adds ", + (false, false) => "sub ", + (false, true) => "subs ", + })?; + gpr(f, a.width, a.rd)?; + f.write_str(", ")?; + } + if let Some(rn) = rn { + gpr(f, a.width, rn)?; + f.write_str(", ")?; + } + gpr(f, a.width, a.rm)?; + shift_operand(f, a.shift, a.amount) +} + +fn arith_ext(f: &mut Formatter<'_>, a: ArithmeticExtended) -> fmt::Result { + let add = a.op == ArithOp::Add; + // `rd` is the stack pointer only where the instruction does not set flags. + let rd_sp = !a.flags && a.rd == 31; + if a.flags && a.rd == 31 { + f.write_str(if add { "cmn " } else { "cmp " })?; + } else { + f.write_str(match (add, a.flags) { + (true, false) => "add ", + (true, true) => "adds ", + (false, false) => "sub ", + (false, true) => "subs ", + })?; + if a.flags { + gpr(f, a.width, a.rd)?; + } else { + gpr_sp(f, a.width, a.rd)?; + } + f.write_str(", ")?; + } + gpr_sp(f, a.width, a.rn)?; + f.write_str(", ")?; + let wide_rm = a.width == Width::X64 && matches!(a.extend, Extend::Uxtx | Extend::Sxtx); + gpr(f, if wide_rm { Width::X64 } else { Width::W32 }, a.rm)?; + let native = match a.width { + Width::W32 => Extend::Uxtw, + Width::X64 => Extend::Uxtx, + }; + if (rd_sp || a.rn == 31) && a.extend == native { + lsl_suffix(f, a.amount) + } else { + write!(f, ", {}", extend_name(a.extend))?; + if a.amount != 0 { + write!(f, " #{}", a.amount)?; + } + Ok(()) + } +} + +fn logic_reg(f: &mut Formatter<'_>, l: Logic) -> fmt::Result { + let plain_shift = l.shift == Shift::Lsl && l.amount == 0; + if l.op == LogicOp::BitOr && !l.flags && l.rn == 31 { + if !l.invert && plain_shift { + f.write_str("mov ")?; + gpr(f, l.width, l.rd)?; + f.write_str(", ")?; + return gpr(f, l.width, l.rm); + } + if l.invert { + f.write_str("mvn ")?; + gpr(f, l.width, l.rd)?; + f.write_str(", ")?; + gpr(f, l.width, l.rm)?; + return shift_operand(f, l.shift, l.amount); + } + } + if l.op == LogicOp::BitAnd && l.flags && !l.invert && l.rd == 31 { + f.write_str("tst ")?; + } else { + f.write_str(match (l.op, l.invert, l.flags) { + (LogicOp::BitAnd, false, false) => "and ", + (LogicOp::BitAnd, false, true) => "ands ", + (LogicOp::BitAnd, true, false) => "bic ", + (LogicOp::BitAnd, true, true) => "bics ", + (LogicOp::BitOr, false, _) => "orr ", + (LogicOp::BitOr, true, _) => "orn ", + (LogicOp::BitXor, false, _) => "eor ", + (LogicOp::BitXor, true, _) => "eon ", + })?; + gpr(f, l.width, l.rd)?; + f.write_str(", ")?; + } + gpr(f, l.width, l.rn)?; + f.write_str(", ")?; + gpr(f, l.width, l.rm)?; + shift_operand(f, l.shift, l.amount) +} + +/// Whether a single `movz` or `movn` can load `value` (already masked to `width`). +fn wide_constant(value: u64, width: Width) -> bool { + let single_chunk = + |v: u64| v != 0 && [0, 16, 32, 48].iter().any(|&s| v & !(0xffffu64 << s) == 0); + single_chunk(value) || single_chunk(!value & mask(width)) +} + +fn logic_imm(f: &mut Formatter<'_>, l: LogicImm) -> fmt::Result { + let value = l.immediate & mask(l.width); + if l.op == LogicOp::BitOr + && !l.flags + && l.rn == 31 + && (l.rd == 31 || !wide_constant(value, l.width)) + { + f.write_str("mov ")?; + gpr_sp(f, l.width, l.rd)?; + return write!(f, ", #{value:#x}"); + } + if l.op == LogicOp::BitAnd && l.flags && l.rd == 31 { + f.write_str("tst ")?; + } else { + f.write_str(match (l.op, l.flags) { + (LogicOp::BitAnd, false) => "and ", + (LogicOp::BitAnd, true) => "ands ", + (LogicOp::BitOr, _) => "orr ", + (LogicOp::BitXor, _) => "eor ", + })?; + if l.flags { + gpr(f, l.width, l.rd)?; + } else { + gpr_sp(f, l.width, l.rd)?; + } + f.write_str(", ")?; + } + gpr(f, l.width, l.rn)?; + write!(f, ", #{value:#x}") +} + +fn mul(f: &mut Formatter<'_>, m: Multiply) -> fmt::Result { + let add = m.op == ArithOp::Add; + let alias = m.ra == 31; + f.write_str(match (add, alias) { + (true, true) => "mul ", + (false, true) => "mneg ", + (true, false) => "madd ", + (false, false) => "msub ", + })?; + gpr(f, m.width, m.rd)?; + f.write_str(", ")?; + gpr(f, m.width, m.rn)?; + f.write_str(", ")?; + gpr(f, m.width, m.rm)?; + if !alias { + f.write_str(", ")?; + gpr(f, m.width, m.ra)?; + } + Ok(()) +} + +fn mul_long(f: &mut Formatter<'_>, m: MultiplyLong) -> fmt::Result { + let add = m.op == ArithOp::Add; + let alias = m.ra == 31; + f.write_str(match (m.signed, add, alias) { + (true, true, true) => "smull ", + (true, false, true) => "smnegl ", + (true, true, false) => "smaddl ", + (true, false, false) => "smsubl ", + (false, true, true) => "umull ", + (false, false, true) => "umnegl ", + (false, true, false) => "umaddl ", + (false, false, false) => "umsubl ", + })?; + gpr(f, Width::X64, m.rd)?; + f.write_str(", ")?; + gpr(f, Width::W32, m.rn)?; + f.write_str(", ")?; + gpr(f, Width::W32, m.rm)?; + if !alias { + f.write_str(", ")?; + gpr(f, Width::X64, m.ra)?; + } + Ok(()) +} + +fn mul_high(f: &mut Formatter<'_>, m: MultiplyHigh) -> fmt::Result { + f.write_str(if m.signed { "smulh " } else { "umulh " })?; + gpr(f, Width::X64, m.rd)?; + f.write_str(", ")?; + gpr(f, Width::X64, m.rn)?; + f.write_str(", ")?; + gpr(f, Width::X64, m.rm) +} + +fn adr(f: &mut Formatter<'_>, cx: &Context<'_>, a: Address) -> fmt::Result { + f.write_str(if a.page { "adrp " } else { "adr " })?; + gpr(f, Width::X64, a.rd)?; + f.write_str(", ")?; + if a.page { + cx.page_target(f, a.offset) + } else { + cx.target(f, a.offset) + } +} + +fn csel(f: &mut Formatter<'_>, s: Select) -> fmt::Result { + let real = !matches!(s.cond, Condition::Al | Condition::Nv); + let same = s.rn == s.rm; + // opc: 0 csel, 1 csinc, 2 csinv, 3 csneg. + if real && same { + let alias = match s.opc { + 1 if s.rn == 31 => Some(("cset ", false)), + 1 => Some(("cinc ", true)), + 2 if s.rn == 31 => Some(("csetm ", false)), + 2 => Some(("cinv ", true)), + 3 => Some(("cneg ", true)), + _ => None, + }; + if let Some((name, with_rn)) = alias { + f.write_str(name)?; + gpr(f, s.width, s.rd)?; + f.write_str(", ")?; + if with_rn { + gpr(f, s.width, s.rn)?; + f.write_str(", ")?; + } + return f.write_str(condition(invert(s.cond))); + } + } + f.write_str(match s.opc { + 0 => "csel ", + 1 => "csinc ", + 2 => "csinv ", + _ => "csneg ", + })?; + gpr(f, s.width, s.rd)?; + f.write_str(", ")?; + gpr(f, s.width, s.rn)?; + f.write_str(", ")?; + gpr(f, s.width, s.rm)?; + write!(f, ", {}", condition(s.cond)) +} + +fn variable(f: &mut Formatter<'_>, v: Variable) -> fmt::Result { + f.write_str(match v.op { + VariableOp::Udiv => "udiv ", + VariableOp::Sdiv => "sdiv ", + VariableOp::Lsl => "lsl ", + VariableOp::Lsr => "lsr ", + VariableOp::Asr => "asr ", + VariableOp::Ror => "ror ", + })?; + gpr(f, v.width, v.rd)?; + f.write_str(", ")?; + gpr(f, v.width, v.rn)?; + f.write_str(", ")?; + gpr(f, v.width, v.rm) +} + +fn cond_compare(f: &mut Formatter<'_>, c: CondCompare) -> fmt::Result { + f.write_str(if c.op == ArithOp::Add { + "ccmn " + } else { + "ccmp " + })?; + gpr(f, c.width, c.rn)?; + f.write_str(", ")?; + match c.operand { + CompareOperand::Register(rm) => gpr(f, c.width, rm)?, + CompareOperand::Immediate(imm) => write!(f, "#{imm:#x}")?, + } + write!(f, ", #{:#x}, {}", c.nzcv, condition(c.cond)) +} + +fn arith_carry(f: &mut Formatter<'_>, a: AddCarry) -> fmt::Result { + if a.op == ArithOp::Sub && a.rn == 31 { + f.write_str(if a.flags { "ngcs " } else { "ngc " })?; + gpr(f, a.width, a.rd)?; + f.write_str(", ")?; + return gpr(f, a.width, a.rm); + } + f.write_str(match (a.op, a.flags) { + (ArithOp::Add, false) => "adc ", + (ArithOp::Add, true) => "adcs ", + (ArithOp::Sub, false) => "sbc ", + (ArithOp::Sub, true) => "sbcs ", + })?; + gpr(f, a.width, a.rd)?; + f.write_str(", ")?; + gpr(f, a.width, a.rn)?; + f.write_str(", ")?; + gpr(f, a.width, a.rm) +} + +fn unary(f: &mut Formatter<'_>, u: Unary) -> fmt::Result { + f.write_str(match u.op { + UnaryOp::Rbit => "rbit ", + UnaryOp::Rev16 => "rev16 ", + UnaryOp::Rev32 => "rev32 ", + UnaryOp::Rev => "rev ", + UnaryOp::Clz => "clz ", + })?; + gpr(f, u.width, u.rd)?; + f.write_str(", ")?; + gpr(f, u.width, u.rn) +} + +/// `name rd, rn, #a` or `name rd, rn, #a, #b`. +fn bitfield_alias( + f: &mut Formatter<'_>, + name: &str, + b: Bitfield, + first: u16, + second: Option, +) -> fmt::Result { + write!(f, "{name} ")?; + gpr(f, b.width, b.rd)?; + f.write_str(", ")?; + gpr(f, b.width, b.rn)?; + write!(f, ", #{first}")?; + if let Some(second) = second { + write!(f, ", #{second}")?; + } + Ok(()) +} + +fn bitfield(f: &mut Formatter<'_>, b: Bitfield) -> fmt::Result { + let lsb = u16::from(b.lsb); + let len = u16::from(b.len); + let top = lsb.wrapping_add(len) == bits(b.width); + match b.op { + BitfieldOp::Signed => { + if b.extract && lsb == 0 { + let ext = match (len, b.width) { + (8, _) => Some("sxtb "), + (16, _) => Some("sxth "), + (32, Width::X64) => Some("sxtw "), + _ => None, + }; + if let Some(name) = ext { + f.write_str(name)?; + gpr(f, b.width, b.rd)?; + f.write_str(", ")?; + return gpr(f, Width::W32, b.rn); + } + } + if b.extract && top { + bitfield_alias(f, "asr", b, lsb, None) + } else if b.extract { + bitfield_alias(f, "sbfx", b, lsb, Some(len)) + } else { + bitfield_alias(f, "sbfiz", b, lsb, Some(len)) + } + } + BitfieldOp::Unsigned => { + if b.extract && lsb == 0 && b.width == Width::W32 && (len == 8 || len == 16) { + f.write_str(if len == 8 { "uxtb " } else { "uxth " })?; + gpr(f, b.width, b.rd)?; + f.write_str(", ")?; + return gpr(f, b.width, b.rn); + } + if !b.extract && top { + bitfield_alias(f, "lsl", b, lsb, None) + } else if b.extract && top { + bitfield_alias(f, "lsr", b, lsb, None) + } else if b.extract { + bitfield_alias(f, "ubfx", b, lsb, Some(len)) + } else { + bitfield_alias(f, "ubfiz", b, lsb, Some(len)) + } + } + BitfieldOp::Insert => { + if !b.extract && b.rn == 31 { + write!(f, "bfc ")?; + gpr(f, b.width, b.rd)?; + write!(f, ", #{lsb}, #{len}") + } else if b.extract { + bitfield_alias(f, "bfxil", b, lsb, Some(len)) + } else { + bitfield_alias(f, "bfi", b, lsb, Some(len)) + } + } + } +} + +fn extract(f: &mut Formatter<'_>, e: Extract) -> fmt::Result { + let ror = e.rn == e.rm; + f.write_str(if ror { "ror " } else { "extr " })?; + gpr(f, e.width, e.rd)?; + f.write_str(", ")?; + gpr(f, e.width, e.rn)?; + if !ror { + f.write_str(", ")?; + gpr(f, e.width, e.rm)?; + } + write!(f, ", #{}", e.lsb) +} + +#[cfg(test)] +mod tests { + extern crate std; + use super::*; + use crate::decode::decode; + use core::fmt::Write as _; + use std::string::String; + + struct At<'a>(&'a Instruction, Option); + + impl core::fmt::Display for At<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let cx = Context { + pc: self.1, + symbols: &(), + }; + write(f, &cx, self.0) + } + } + + fn text(word: u32, pc: Option) -> String { + let instruction = decode(word).expect("decodes"); + let mut out = String::new(); + write!(out, "{}", At(&instruction, pc)).unwrap(); + out + } + + /// Words and their GNU objdump rendering. + const CASES: &[(u32, &str)] = &[ + (0xd2800000, "mov x0, #0x0"), + (0xd2a00000, "movz x0, #0x0, lsl #16"), + (0x52a00000, "movz w0, #0x0, lsl #16"), + (0xd2e000a0, "mov x0, #0x5000000000000"), + (0x92800000, "mov x0, #0xffffffffffffffff"), + (0x12800000, "mov w0, #0xffffffff"), + (0x92a00000, "movn x0, #0x0, lsl #16"), + (0x12a00000, "movn w0, #0x0, lsl #16"), + (0x129fffe0, "movn w0, #0xffff"), + (0x12bfffe0, "movn w0, #0xffff, lsl #16"), + (0x929fffe0, "mov x0, #0xffffffffffff0000"), + (0x12a000a0, "mov w0, #0xfffaffff"), + (0xf28000a0, "movk x0, #0x5"), + (0x72a000a0, "movk w0, #0x5, lsl #16"), + (0xf2ffffe0, "movk x0, #0xffff, lsl #48"), + (0x91400420, "add x0, x1, #0x1, lsl #12"), + (0x917ffc20, "add x0, x1, #0xfff, lsl #12"), + (0x91400020, "add x0, x1, #0x0, lsl #12"), + (0x9140003f, "add sp, x1, #0x0, lsl #12"), + (0x914003e0, "add x0, sp, #0x0, lsl #12"), + (0xd140003f, "sub sp, x1, #0x0, lsl #12"), + (0xf140003f, "cmp x1, #0x0, lsl #12"), + (0xb140003f, "cmn x1, #0x0, lsl #12"), + (0x11400c20, "add w0, w1, #0x3, lsl #12"), + (0x7140001f, "cmp w0, #0x0, lsl #12"), + (0x9100003f, "mov sp, x1"), + (0x910003e0, "mov x0, sp"), + (0x91000020, "add x0, x1, #0x0"), + (0xb10003e0, "adds x0, sp, #0x0"), + (0xb1000c3f, "cmn x1, #0x3"), + (0x71000fff, "cmp wsp, #0x3"), + (0xf1000fe0, "subs x0, sp, #0x3"), + (0xd10043ff, "sub sp, sp, #0x10"), + (0xcb0103e0, "neg x0, x1"), + (0xeb010fe0, "negs x0, x1, lsl #3"), + (0xeb0103ff, "cmp xzr, x1"), + (0xab82103f, "cmn x1, x2, asr #4"), + (0xab0203ff, "cmn xzr, x2"), + (0x4b810fe0, "neg w0, w1, asr #3"), + (0x8b020020, "add x0, x1, x2"), + (0x8b224020, "add x0, x1, w2, uxtw"), + (0x8b224820, "add x0, x1, w2, uxtw #2"), + (0x8b226020, "add x0, x1, x2, uxtx"), + (0x8b226c20, "add x0, x1, x2, uxtx #3"), + (0x8b22e420, "add x0, x1, x2, sxtx #1"), + (0x8b22d020, "add x0, x1, w2, sxtw #4"), + (0x8b2263ff, "add sp, sp, x2"), + (0x8b226bff, "add sp, sp, x2, lsl #2"), + (0x8b226c3f, "add sp, x1, x2, lsl #3"), + (0x8b226fe0, "add x0, sp, x2, lsl #3"), + (0x0b2247e0, "add w0, wsp, w2, lsl #1"), + (0x0b2243e0, "add w0, wsp, w2"), + (0x0b226020, "add w0, w1, w2, uxtx"), + (0x0b224020, "add w0, w1, w2, uxtw"), + (0x0b228820, "add w0, w1, w2, sxtb #2"), + (0xab2267e0, "adds x0, sp, x2, lsl #1"), + (0xab2263e0, "adds x0, sp, x2"), + (0xab2267ff, "cmn sp, x2, lsl #1"), + (0xab22643f, "cmn x1, x2, uxtx #1"), + (0xeb2203ff, "cmp sp, w2, uxtb"), + (0x6b22a7ff, "cmp wsp, w2, sxth #1"), + (0xab22c03f, "cmn x1, w2, sxtw"), + (0xcb22703f, "sub sp, x1, x2, lsl #4"), + (0xcb228020, "sub x0, x1, w2, sxtb"), + (0x129fffc0, "mov w0, #0xffff0001"), + (0x12bfffc0, "mov w0, #0x1ffff"), + (0x92bfffe0, "mov x0, #0xffffffff0000ffff"), + (0x92dfffe0, "mov x0, #0xffff0000ffffffff"), + (0x92ffffe0, "mov x0, #0xffffffffffff"), + (0x92e00020, "mov x0, #0xfffeffffffffffff"), + (0x52bfffe0, "mov w0, #0xffff0000"), + (0x529fffe0, "mov w0, #0xffff"), + (0xd2ffffe0, "mov x0, #0xffff000000000000"), + (0x8a020020, "and x0, x1, x2"), + (0x8a020c20, "and x0, x1, x2, lsl #3"), + (0x8ac20020, "and x0, x1, x2, ror #0"), + (0x8ac20c20, "and x0, x1, x2, ror #3"), + (0xaa0203e0, "mov x0, x2"), + (0xaa020fe0, "orr x0, xzr, x2, lsl #3"), + (0xaa1f0020, "orr x0, x1, xzr"), + (0x2a0203e0, "mov w0, w2"), + (0xaa1f03e0, "mov x0, xzr"), + (0xaa2203e0, "mvn x0, x2"), + (0xaa6213e0, "mvn x0, x2, lsr #4"), + (0xaa220020, "orn x0, x1, x2"), + (0xca2203e0, "eon x0, xzr, x2"), + (0xca0203e0, "eor x0, xzr, x2"), + (0x8a220020, "bic x0, x1, x2"), + (0xea220020, "bics x0, x1, x2"), + (0xea22003f, "bics xzr, x1, x2"), + (0xea02003f, "tst x1, x2"), + (0xea02143f, "tst x1, x2, lsl #5"), + (0xea0203ff, "tst xzr, x2"), + (0xea020020, "ands x0, x1, x2"), + (0xaae203e0, "mvn x0, x2, ror #0"), + (0xaac203e0, "orr x0, xzr, x2, ror #0"), + (0xaa8207e0, "orr x0, xzr, x2, asr #1"), + (0x92401c20, "and x0, x1, #0xff"), + (0x92401c3f, "and sp, x1, #0xff"), + (0xb2401fe0, "orr x0, xzr, #0xff"), + (0xb200f3e0, "mov x0, #0x5555555555555555"), + (0x32089fe0, "mov w0, #0xff00ff00"), + (0x320003e0, "orr w0, wzr, #0x1"), + (0x320003ff, "mov wsp, #0x1"), + (0xb24003ff, "mov sp, #0x1"), + (0xb278dfe0, "orr x0, xzr, #0xffffffffffffff00"), + (0xb2410020, "orr x0, x1, #0x8000000000000000"), + (0x321f7820, "orr w0, w1, #0xfffffffe"), + (0xf27c0c3f, "tst x1, #0xf0"), + (0xf27c0c20, "ands x0, x1, #0xf0"), + (0x721c0c3f, "tst w1, #0xf0"), + (0xd2401fe0, "eor x0, xzr, #0xff"), + (0x52007820, "eor w0, w1, #0x7fffffff"), + (0xb27f7be0, "mov x0, #0xfffffffe"), + (0xb240fbe0, "orr x0, xzr, #0x7fffffffffffffff"), + (0x321f7be0, "orr w0, wzr, #0xfffffffe"), + (0x9b027c20, "mul x0, x1, x2"), + (0x9b020c20, "madd x0, x1, x2, x3"), + (0x9b02fc20, "mneg x0, x1, x2"), + (0x9b028c20, "msub x0, x1, x2, x3"), + (0x1b02fc20, "mneg w0, w1, w2"), + (0x1b027c20, "mul w0, w1, w2"), + (0x9b220c20, "smaddl x0, w1, w2, x3"), + (0x9b227c20, "smull x0, w1, w2"), + (0x9b228c20, "smsubl x0, w1, w2, x3"), + (0x9b22fc20, "smnegl x0, w1, w2"), + (0x9ba20c20, "umaddl x0, w1, w2, x3"), + (0x9ba27c20, "umull x0, w1, w2"), + (0x9ba28c20, "umsubl x0, w1, w2, x3"), + (0x9ba2fc20, "umnegl x0, w1, w2"), + (0x9b427c20, "smulh x0, x1, x2"), + (0x9bc27c20, "umulh x0, x1, x2"), + (0x9a820020, "csel x0, x1, x2, eq"), + (0x9a82e020, "csel x0, x1, x2, al"), + (0x9a82f020, "csel x0, x1, x2, nv"), + (0x9a821420, "csinc x0, x1, x2, ne"), + (0x9a811420, "cinc x0, x1, eq"), + (0x9a81e420, "csinc x0, x1, x1, al"), + (0x9a81f420, "csinc x0, x1, x1, nv"), + (0x9a9f17e0, "cset x0, eq"), + (0x9a9fe7e0, "csinc x0, xzr, xzr, al"), + (0x9a9ff7e0, "csinc x0, xzr, xzr, nv"), + (0x9a8117e0, "csinc x0, xzr, x1, ne"), + (0x9a9f1420, "csinc x0, x1, xzr, ne"), + (0xda9f13e0, "csetm x0, eq"), + (0xda9fe3e0, "csinv x0, xzr, xzr, al"), + (0xda811020, "cinv x0, x1, eq"), + (0xda81f020, "csinv x0, x1, x1, nv"), + (0xda8113e0, "csinv x0, xzr, x1, ne"), + (0xda811420, "cneg x0, x1, eq"), + (0xda81e420, "csneg x0, x1, x1, al"), + (0xda9f17e0, "cneg x0, xzr, eq"), + (0xda9fe7e0, "csneg x0, xzr, xzr, al"), + (0xda821420, "csneg x0, x1, x2, ne"), + (0x1a9f27e0, "cset w0, cc"), + (0x1a9f87e0, "cset w0, ls"), + (0x1a9f47e0, "cset w0, pl"), + (0x1a9f77e0, "cset w0, vs"), + (0x1a9fa7e0, "cset w0, lt"), + (0x1a9fd7e0, "cset w0, gt"), + (0x9ac20820, "udiv x0, x1, x2"), + (0x1ac20c20, "sdiv w0, w1, w2"), + (0x9ac22020, "lsl x0, x1, x2"), + (0x1ac22420, "lsr w0, w1, w2"), + (0x9ac22820, "asr x0, x1, x2"), + (0x1ac22c20, "ror w0, w1, w2"), + (0xfa410005, "ccmp x0, x1, #0x5, eq"), + (0xfa5f180f, "ccmp x0, #0x1f, #0xf, ne"), + (0x3a41e000, "ccmn w0, w1, #0x0, al"), + (0x3a40f800, "ccmn w0, #0x0, #0x0, nv"), + (0xfa5f23e1, "ccmp xzr, xzr, #0x1, cs"), + (0x9a020020, "adc x0, x1, x2"), + (0x3a020020, "adcs w0, w1, w2"), + (0xda020020, "sbc x0, x1, x2"), + (0xda0203e0, "ngc x0, x2"), + (0xfa0203e0, "ngcs x0, x2"), + (0x7a0203e0, "ngcs w0, w2"), + (0x5a1f03e0, "ngc w0, wzr"), + (0x9a1f03e0, "adc x0, xzr, xzr"), + (0xdac00020, "rbit x0, x1"), + (0x5ac00020, "rbit w0, w1"), + (0xdac00420, "rev16 x0, x1"), + (0x5ac00420, "rev16 w0, w1"), + (0xdac00820, "rev32 x0, x1"), + (0x5ac00820, "rev w0, w1"), + (0xdac00c20, "rev x0, x1"), + (0xdac01020, "clz x0, x1"), + (0x5ac013e0, "clz w0, wzr"), + (0x93401c20, "sxtb x0, w1"), + (0x93403c20, "sxth x0, w1"), + (0x93407c20, "sxtw x0, w1"), + (0x13001c20, "sxtb w0, w1"), + (0x13003c20, "sxth w0, w1"), + (0x13007c20, "asr w0, w1, #0"), + (0x9340fc20, "asr x0, x1, #0"), + (0x9345fc20, "asr x0, x1, #5"), + (0x13057c20, "asr w0, w1, #5"), + (0x93452420, "sbfx x0, x1, #5, #5"), + (0x93491420, "sbfiz x0, x1, #55, #6"), + (0x937ff820, "sbfiz x0, x1, #1, #63"), + (0x131f7820, "sbfiz w0, w1, #1, #31"), + (0x93410020, "sbfiz x0, x1, #63, #1"), + (0x93400020, "sbfx x0, x1, #0, #1"), + (0x93402020, "sbfx x0, x1, #0, #9"), + (0x93430c20, "sbfx x0, x1, #3, #1"), + (0xd3401c20, "ubfx x0, x1, #0, #8"), + (0x53001c20, "uxtb w0, w1"), + (0x53003c20, "uxth w0, w1"), + (0xd3403c20, "ubfx x0, x1, #0, #16"), + (0xd3407c20, "ubfx x0, x1, #0, #32"), + (0xd340fc20, "lsr x0, x1, #0"), + (0x53007c20, "lsr w0, w1, #0"), + (0xd345fc20, "lsr x0, x1, #5"), + (0x53057c20, "lsr w0, w1, #5"), + (0xd3452420, "ubfx x0, x1, #5, #5"), + (0xd3491420, "ubfiz x0, x1, #55, #6"), + (0xd37ff820, "lsl x0, x1, #1"), + (0x531f7820, "lsl w0, w1, #1"), + (0xd3410020, "lsl x0, x1, #63"), + (0xd3400020, "ubfx x0, x1, #0, #1"), + (0x53010020, "lsl w0, w1, #31"), + (0xd3430c20, "ubfx x0, x1, #3, #1"), + (0xd340f820, "ubfx x0, x1, #0, #63"), + (0xd37df020, "lsl x0, x1, #3"), + (0xb3452420, "bfxil x0, x1, #5, #5"), + (0xb3491420, "bfi x0, x1, #55, #6"), + (0xb34917e0, "bfc x0, #55, #6"), + (0xb34527e0, "bfxil x0, xzr, #5, #5"), + (0xb34027e0, "bfxil x0, xzr, #0, #10"), + (0xb3400020, "bfxil x0, x1, #0, #1"), + (0xb37ff820, "bfi x0, x1, #1, #63"), + (0x330103e0, "bfc w0, #31, #1"), + (0x33007c20, "bfxil w0, w1, #0, #32"), + (0xb340fc20, "bfxil x0, x1, #0, #64"), + (0xb340ffe0, "bfxil x0, xzr, #0, #64"), + (0x93c21420, "extr x0, x1, x2, #5"), + (0x93c11420, "ror x0, x1, #5"), + (0x13810020, "ror w0, w1, #0"), + (0x93c20020, "extr x0, x1, x2, #0"), + (0x13827c20, "extr w0, w1, w2, #31"), + (0x93dfffe0, "ror x0, xzr, #63"), + ]; + + #[test] + fn matches_objdump() { + for &(word, expected) in CASES { + assert_eq!(text(word, None), expected, "{word:#010x}"); + } + } + + #[test] + fn pc_relative() { + assert_eq!(text(0x1000_0020, None), "adr x0, .+4"); + assert_eq!(text(0x10ff_ffe0, None), "adr x0, .-4"); + assert_eq!(text(0xb000_0000, None), "adrp x0, .+4096"); + assert_eq!(text(0x90ff_ffe0, None), "adrp x0, .-16384"); + assert_eq!(text(0x1000_0020, Some(0x1000)), "adr x0, 0x1004"); + assert_eq!(text(0xb000_0000, Some(0x1234)), "adrp x0, 0x2000"); + assert_eq!(text(0xb000_001f, Some(0x1234)), "adrp xzr, 0x2000"); + } +} diff --git a/src/isa/aarch64/src/format/memory.rs b/src/isa/aarch64/src/format/memory.rs new file mode 100644 index 0000000..1ee15c0 --- /dev/null +++ b/src/isa/aarch64/src/format/memory.rs @@ -0,0 +1,870 @@ +//! Loads and stores: scalar, pair, literal, exclusive, LSE atomics, prefetch and the +//! SIMD&FP and structure forms. +//! +//! Owns `Memory`, `Pair`, `Literal`, `SimdLiteral`, `Exclusive`, `Atomic`, `Prefetch`, +//! `PrefetchLiteral`, `SimdPair`, `SimdMemory` and `SimdStruct`. +//! +//! The decoded forms keep what distinguishes the instructions: `Memory::ordered` for +//! `ldar`/`stlr`/`ldapr`/`ldlar`/`stllr`, `MemoryOrder` for the `a`/`l` of exclusives and +//! atomics, `Addressing::Unscaled` for `ldur`/`prfum`, `Pair::non_temporal` for `ldnp` and +//! `SimdAccess::Ld1` for a one-register `ld1`/`st1`. + +use super::Context; +use super::reg::{fp_scalar, gpr, gpr_sp, vector}; +use crate::decode::{ + Addressing, Atomic, AtomicKind, Exclusive, Extend, Instruction, Literal, Memory, MemoryOp, + MemoryOrder, Ordered, Pair, Prefetch, PrefetchLiteral, RangePrefetch, SignExtend, SimdAccess, + SimdLiteral, SimdMemory, SimdPair, SimdStruct, Size, StructPost, Width, +}; +use crate::simd_struct::{Shape, StructDesc}; +use core::fmt::{self, Formatter}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + match instruction { + Instruction::Memory(i) => memory(f, i), + Instruction::Pair(i) => pair(f, i), + Instruction::Literal(i) => literal(f, cx, i), + Instruction::SimdLiteral(i) => simd_literal(f, cx, i), + Instruction::Exclusive(i) => exclusive(f, i), + Instruction::Atomic(i) => atomic(f, i), + Instruction::Prefetch(i) => prefetch(f, i), + Instruction::PrefetchLiteral(i) => prefetch_literal(f, cx, i), + Instruction::RangePrefetch(i) => range_prefetch(f, i), + Instruction::SimdPair(i) => simd_pair(f, i), + Instruction::SimdMemory(i) => simd_memory(f, i), + Instruction::SimdStruct(i) => simd_struct(f, i), + _ => unreachable!("{instruction:?} is not a memory instruction"), + } +} + +fn width_of(size: Size) -> Width { + if size == Size::Double { + Width::X64 + } else { + Width::W32 + } +} + +/// `[xn]`, `[xn, #imm]`, `[xn, #imm]!`, `[xn], #imm` or `[xn, rm{, ext{ #n}}]`. +fn address(f: &mut Formatter<'_>, rn: u8, addressing: Addressing) -> fmt::Result { + f.write_str("[")?; + gpr_sp(f, Width::X64, rn)?; + match addressing { + Addressing::Offset(0) | Addressing::Unscaled(0) | Addressing::Unprivileged(0) => { + f.write_str("]") + } + Addressing::Offset(offset) + | Addressing::Unscaled(offset) + | Addressing::Unprivileged(offset) => write!(f, ", #{offset}]"), + Addressing::PreIndex(offset) => write!(f, ", #{offset}]!"), + Addressing::PostIndex(offset) => write!(f, "], #{offset}"), + Addressing::Register { rm, extend, amount } => { + f.write_str(", ")?; + let wide = matches!(extend, Extend::Uxtx | Extend::Sxtx); + gpr(f, if wide { Width::X64 } else { Width::W32 }, rm)?; + let name = match (extend, amount) { + (Extend::Uxtx, None) => return f.write_str("]"), + (Extend::Uxtx, Some(_)) => "lsl", + (Extend::Uxtb, _) => "uxtb", + (Extend::Uxth, _) => "uxth", + (Extend::Uxtw, _) => "uxtw", + (Extend::Sxtb, _) => "sxtb", + (Extend::Sxth, _) => "sxth", + (Extend::Sxtw, _) => "sxtw", + (Extend::Sxtx, _) => "sxtx", + }; + write!(f, ", {name}")?; + if let Some(amount) = amount { + write!(f, " #{amount}")?; + } + f.write_str("]") + } + } +} + +fn memory(f: &mut Formatter<'_>, i: &Memory) -> fmt::Result { + let (op, suffix) = match (i.op, i.size, i.signed) { + (MemoryOp::Load, Size::Byte, SignExtend::None) => ("ld", "b"), + (MemoryOp::Load, Size::Half, SignExtend::None) => ("ld", "h"), + (MemoryOp::Load, Size::Byte, _) => ("ld", "sb"), + (MemoryOp::Load, Size::Half, _) => ("ld", "sh"), + (MemoryOp::Load, Size::Word, SignExtend::None) => ("ld", ""), + (MemoryOp::Load, Size::Word, _) => ("ld", "sw"), + (MemoryOp::Load, Size::Double, _) => ("ld", ""), + (MemoryOp::Store, Size::Byte, _) => ("st", "b"), + (MemoryOp::Store, Size::Half, _) => ("st", "h"), + (MemoryOp::Store, _, _) => ("st", ""), + }; + // The middle of the mnemonic: `ldr`, `ldur`, `ldtr`, `ldar`, `ldapr`, `ldapur`, + // `ldlar`, `stlr`, `stlur`, `stllr`. + let unscaled = matches!(i.addressing, Addressing::Unscaled(_)); + let stem = match i.ordered { + None => match i.addressing { + Addressing::Unscaled(_) => "ur", + Addressing::Unprivileged(_) => "tr", + _ => "r", + }, + Some(Ordered::Acquire) => "ar", + Some(Ordered::AcquirePc) if unscaled => "apur", + Some(Ordered::AcquirePc) => "apr", + Some(Ordered::LimitedAcquire) => "lar", + Some(Ordered::Release) if unscaled => "lur", + Some(Ordered::Release) => "lr", + Some(Ordered::LimitedRelease) => "llr", + }; + let width = if i.signed == SignExtend::To64 { + Width::X64 + } else if i.signed == SignExtend::To32 { + Width::W32 + } else { + width_of(i.size) + }; + write!(f, "{op}{stem}{suffix} ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn pair(f: &mut Formatter<'_>, i: &Pair) -> fmt::Result { + let op = match (i.op, i.signed, i.non_temporal) { + (MemoryOp::Load, SignExtend::None, false) => "ldp", + (MemoryOp::Load, SignExtend::None, true) => "ldnp", + (MemoryOp::Load, _, _) => "ldpsw", + (MemoryOp::Store, _, false) => "stp", + (MemoryOp::Store, _, true) => "stnp", + }; + let width = if i.signed != SignExtend::None { + Width::X64 + } else { + width_of(i.size) + }; + write!(f, "{op} ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + gpr(f, width, i.rt2)?; + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn literal(f: &mut Formatter<'_>, cx: &Context<'_>, i: &Literal) -> fmt::Result { + if i.signed == SignExtend::None { + f.write_str("ldr ")?; + gpr(f, width_of(i.size), i.rt)?; + } else { + f.write_str("ldrsw ")?; + gpr(f, Width::X64, i.rt)?; + } + f.write_str(", ")?; + cx.target(f, i.offset) +} + +fn simd_literal(f: &mut Formatter<'_>, cx: &Context<'_>, i: &SimdLiteral) -> fmt::Result { + f.write_str("ldr ")?; + fp_scalar(f, i.bytes, i.rt)?; + f.write_str(", ")?; + cx.target(f, i.offset) +} + +fn exclusive(f: &mut Formatter<'_>, i: &Exclusive) -> fmt::Result { + let (op, suffix) = ( + match (i.op, i.order) { + (MemoryOp::Load, MemoryOrder { acquire: true, .. }) => "ldaxr", + (MemoryOp::Load, _) => "ldxr", + (MemoryOp::Store, MemoryOrder { release: true, .. }) => "stlxr", + (MemoryOp::Store, _) => "stxr", + }, + match i.size { + Size::Byte => "b", + Size::Half => "h", + _ => "", + }, + ); + write!(f, "{op}{suffix} ")?; + if i.op == MemoryOp::Store { + gpr(f, Width::W32, i.rs)?; + f.write_str(", ")?; + } + gpr(f, width_of(i.size), i.rt)?; + f.write_str(", ")?; + address(f, i.rn, Addressing::Offset(0)) +} + +fn atomic(f: &mut Formatter<'_>, i: &Atomic) -> fmt::Result { + let width = width_of(i.size); + let suffix = match i.size { + Size::Byte => "b", + Size::Half => "h", + _ => "", + }; + let a = if i.order.acquire { "a" } else { "" }; + let l = if i.order.release { "l" } else { "" }; + let op = match i.kind { + AtomicKind::Add => "add", + AtomicKind::Clr => "clr", + AtomicKind::Eor => "eor", + AtomicKind::Set => "set", + AtomicKind::SMax => "smax", + AtomicKind::SMin => "smin", + AtomicKind::UMax => "umax", + AtomicKind::UMin => "umin", + AtomicKind::Swap | AtomicKind::Cas | AtomicKind::Casp => "", + AtomicKind::LoadPair | AtomicKind::StorePair => "", + }; + let memory = Addressing::Offset(0); + match i.kind { + AtomicKind::Swap => { + write!(f, "swp{a}{l}{suffix} ")?; + gpr(f, width, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + } + AtomicKind::Cas => { + write!(f, "cas{a}{l}{suffix} ")?; + gpr(f, width, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + } + AtomicKind::Casp => { + write!(f, "casp{a}{l} ")?; + gpr(f, width, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rs.wrapping_add(1) & 31)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + gpr(f, width, i.rt.wrapping_add(1) & 31)?; + } + AtomicKind::LoadPair => { + write!(f, "ld{a}xp ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + gpr(f, width, i.rt2)?; + } + AtomicKind::StorePair => { + write!(f, "st{l}xp ")?; + gpr(f, Width::W32, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + gpr(f, width, i.rt2)?; + } + _ if i.rt == 31 && !i.order.acquire => { + // `ldadd Rs, xzr, [Xn]` is `stadd Rs, [Xn]`; with acquire the load must + // happen, so `ldadda Rs, xzr, [Xn]` keeps its spelling. + write!(f, "st{op}{l}{suffix} ")?; + gpr(f, width, i.rs)?; + } + _ => { + write!(f, "ld{op}{a}{l}{suffix} ")?; + gpr(f, width, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + } + } + f.write_str(", ")?; + address(f, i.rn, memory) +} + +/// The operation operand of `prfm`: `pldl1keep`.. or `#0x..` when unnamed. Operation 24 +/// is `ir` only for the scaled-offset form (`ir_form`); every other form spells it `#0x18`. +fn prefetch_operation(f: &mut Formatter<'_>, op: u8, ir_form: bool) -> fmt::Result { + if op == 24 && ir_form { + return f.write_str("ir"); + } + let kind = match op >> 3 { + 0 => "pld", + 1 => "pli", + 2 => "pst", + _ => return write!(f, "#{op:#x}"), + }; + let target = ["l1", "l2", "l3", "slc"][usize::from((op >> 1) & 3)]; + let policy = if op & 1 == 0 { "keep" } else { "strm" }; + write!(f, "{kind}{target}{policy}") +} + +fn prefetch(f: &mut Formatter<'_>, i: &Prefetch) -> fmt::Result { + let u = if matches!(i.addressing, Addressing::Unscaled(_)) { + "u" + } else { + "" + }; + let ir_form = matches!(i.addressing, Addressing::Offset(_)); + write!(f, "prf{u}m ")?; + prefetch_operation(f, i.op, ir_form)?; + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn prefetch_literal(f: &mut Formatter<'_>, cx: &Context<'_>, i: &PrefetchLiteral) -> fmt::Result { + f.write_str("prfm ")?; + prefetch_operation(f, i.op, false)?; + f.write_str(", ")?; + cx.target(f, i.offset) +} + +/// `rprfm pldkeep, xm, [xn]`: the four named operations, `#n` for the rest. +fn range_prefetch(f: &mut Formatter<'_>, i: &RangePrefetch) -> fmt::Result { + f.write_str("rprfm ")?; + match i.op { + 0 => f.write_str("pldkeep")?, + 1 => f.write_str("pstkeep")?, + 4 => f.write_str("pldstrm")?, + 5 => f.write_str("pststrm")?, + n => write!(f, "#{n}")?, + } + f.write_str(", ")?; + gpr(f, Width::X64, i.rm)?; + f.write_str(", ")?; + address(f, i.rn, Addressing::Offset(0)) +} + +fn simd_pair(f: &mut Formatter<'_>, i: &SimdPair) -> fmt::Result { + f.write_str(match (i.op, i.non_temporal) { + (MemoryOp::Load, false) => "ldp ", + (MemoryOp::Store, false) => "stp ", + (MemoryOp::Load, true) => "ldnp ", + (MemoryOp::Store, true) => "stnp ", + })?; + fp_scalar(f, i.bytes, i.rt)?; + f.write_str(", ")?; + fp_scalar(f, i.bytes, i.rt2)?; + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn simd_memory(f: &mut Formatter<'_>, i: &SimdMemory) -> fmt::Result { + let op = match i.op { + MemoryOp::Load => "ld", + MemoryOp::Store => "st", + }; + match i.access { + SimdAccess::Register => { + let u = if matches!(i.addressing, Addressing::Unscaled(_)) { + "u" + } else { + "" + }; + write!(f, "{op}{u}r ")?; + fp_scalar(f, i.bytes, i.rt)?; + } + SimdAccess::Ld1 { esize } => { + write!(f, "{op}1 {{")?; + vector(f, i.rt, esize.trailing_zeros() as u8, i.bytes == 16)?; + f.write_str("}")?; + } + } + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn element(esize: u8) -> char { + match esize { + 1 => 'b', + 2 => 'h', + 4 => 's', + _ => 'd', + } +} + +fn register_list(f: &mut Formatter<'_>, d: &StructDesc) -> fmt::Result { + let size = (d.esize.max(1)).trailing_zeros() as u8; + let last = d.rt.wrapping_add(d.registers.wrapping_sub(1)) & 31; + f.write_str("{")?; + for (n, r) in [d.rt, last].into_iter().enumerate() { + if n == 1 { + if d.registers <= 1 { + break; + } + f.write_str("-")?; + } + match d.shape { + Shape::Lane => write!(f, "v{r}.{}", element(d.esize))?, + _ => vector(f, r, size, d.q)?, + } + } + f.write_str("}")?; + if d.shape == Shape::Lane { + write!(f, "[{}]", d.lane)?; + } + Ok(()) +} + +fn simd_struct(f: &mut Formatter<'_>, i: &SimdStruct) -> fmt::Result { + let d = &i.desc; + let count = if d.shape == Shape::Multiple && !d.interleaved { + 1 + } else { + d.registers + }; + let op = if d.store { "st" } else { "ld" }; + let replicate = if d.shape == Shape::Replicate { "r" } else { "" }; + write!(f, "{op}{count}{replicate} ")?; + register_list(f, d)?; + f.write_str(", [")?; + gpr_sp(f, Width::X64, i.rn)?; + f.write_str("]")?; + match i.post { + StructPost::None => Ok(()), + StructPost::Immediate => write!(f, ", #{}", d.bytes()), + StructPost::Register(rm) => { + f.write_str(", ")?; + gpr(f, Width::X64, rm) + } + } +} + +#[cfg(test)] +mod tests { + use crate::decode::decode; + use std::string::ToString; + + /// `(pc, word, text)`: text is what GNU objdump 2.46 prints for `word` at `pc`. + const ROWS: &[(u64, u32, &str)] = &[ + (0x0, 0xf9400020, "ldr x0, [x1]"), + (0x4, 0xf9400420, "ldr x0, [x1, #8]"), + (0x8, 0xf97ffc20, "ldr x0, [x1, #32760]"), + (0xc, 0xf94007e0, "ldr x0, [sp, #8]"), + (0x10, 0xf85f8020, "ldur x0, [x1, #-8]"), + (0x14, 0xf8401020, "ldur x0, [x1, #1]"), + (0x18, 0xb97ffc20, "ldr w0, [x1, #16380]"), + (0x1c, 0xf8408c20, "ldr x0, [x1, #8]!"), + (0x20, 0xf8500fe0, "ldr x0, [sp, #-256]!"), + (0x24, 0xf84ff420, "ldr x0, [x1], #255"), + (0x28, 0xb85fc422, "ldr w2, [x1], #-4"), + (0x2c, 0xf8626820, "ldr x0, [x1, x2]"), + (0x30, 0xf8627820, "ldr x0, [x1, x2, lsl #3]"), + (0x34, 0xf87f7820, "ldr x0, [x1, xzr, lsl #3]"), + (0x38, 0xf8624820, "ldr x0, [x1, w2, uxtw]"), + (0x3c, 0xf8625820, "ldr x0, [x1, w2, uxtw #3]"), + (0x40, 0xf862c820, "ldr x0, [x1, w2, sxtw]"), + (0x44, 0xf862d820, "ldr x0, [x1, w2, sxtw #3]"), + (0x48, 0xf862e820, "ldr x0, [x1, x2, sxtx]"), + (0x4c, 0xf862f820, "ldr x0, [x1, x2, sxtx #3]"), + (0x50, 0xb8627820, "ldr w0, [x1, x2, lsl #2]"), + (0x54, 0xb87fd820, "ldr w0, [x1, wzr, sxtw #2]"), + (0x58, 0x397ffc20, "ldrb w0, [x1, #4095]"), + (0x5c, 0x385ff020, "ldurb w0, [x1, #-1]"), + (0x60, 0x38626820, "ldrb w0, [x1, x2]"), + (0x64, 0x3862c820, "ldrb w0, [x1, w2, sxtw]"), + (0x68, 0x38401420, "ldrb w0, [x1], #1"), + (0x6c, 0x38401c3f, "ldrb wzr, [x1, #1]!"), + (0x70, 0x797ffc20, "ldrh w0, [x1, #8190]"), + (0x74, 0x78403020, "ldurh w0, [x1, #3]"), + (0x78, 0x78627820, "ldrh w0, [x1, x2, lsl #1]"), + (0x7c, 0x39c01420, "ldrsb w0, [x1, #5]"), + (0x80, 0x39801420, "ldrsb x0, [x1, #5]"), + (0x84, 0x389fb020, "ldursb x0, [x1, #-5]"), + (0x88, 0x38e26820, "ldrsb w0, [x1, x2]"), + (0x8c, 0x79c00c20, "ldrsh w0, [x1, #6]"), + (0x90, 0x79800c20, "ldrsh x0, [x1, #6]"), + (0x94, 0x78807020, "ldursh x0, [x1, #7]"), + (0x98, 0x78a27820, "ldrsh x0, [x1, x2, lsl #1]"), + (0x9c, 0xb9800420, "ldrsw x0, [x1, #4]"), + (0xa0, 0xb89fc020, "ldursw x0, [x1, #-4]"), + (0xa4, 0xb8804c20, "ldrsw x0, [x1, #4]!"), + (0xa8, 0xb8804420, "ldrsw x0, [x1], #4"), + (0xac, 0xb8a27820, "ldrsw x0, [x1, x2, lsl #2]"), + (0xb0, 0xf9000020, "str x0, [x1]"), + (0xb4, 0xf9000bff, "str xzr, [sp, #16]"), + (0xb8, 0xb81fc03f, "stur wzr, [x1, #-4]"), + (0xbc, 0xb81fcc20, "str w0, [x1, #-4]!"), + (0xc0, 0xf8008420, "str x0, [x1], #8"), + (0xc4, 0xf8227820, "str x0, [x1, x2, lsl #3]"), + (0xc8, 0x39000420, "strb w0, [x1, #1]"), + (0xcc, 0x381ff020, "sturb w0, [x1, #-1]"), + (0xd0, 0x38226820, "strb w0, [x1, x2]"), + (0xd4, 0x79000420, "strh w0, [x1, #2]"), + (0xd8, 0x78001020, "sturh w0, [x1, #1]"), + (0xdc, 0x781fe43f, "strh wzr, [x1], #-2"), + (0xe0, 0xa9400440, "ldp x0, x1, [x2]"), + (0xe4, 0xa9408440, "ldp x0, x1, [x2, #8]"), + (0xe8, 0xa8c17bfd, "ldp x29, x30, [sp], #16"), + (0xec, 0xa9bf7bfd, "stp x29, x30, [sp, #-16]!"), + (0xf0, 0xa91ffbfd, "stp x29, x30, [sp, #504]"), + (0xf4, 0xa9207bfd, "stp x29, x30, [sp, #-512]"), + (0xf8, 0x295f8440, "ldp w0, w1, [x2, #252]"), + (0xfc, 0x29600440, "ldp w0, w1, [x2, #-256]"), + (0x100, 0x28817c40, "stp w0, wzr, [x2], #8"), + (0x104, 0x69400440, "ldpsw x0, x1, [x2]"), + (0x108, 0x697f8440, "ldpsw x0, x1, [x2, #-4]"), + (0x10c, 0x68df8440, "ldpsw x0, x1, [x2], #252"), + (0x110, 0x18000040, "ldr w0, 0x118"), + (0x114, 0x58ffffc0, "ldr x0, 0x10c"), + (0x118, 0x587fffff, "ldr xzr, 0x100114"), + (0x11c, 0x58800000, "ldr x0, 0xfffffffffff0011c"), + (0x120, 0x1c000040, "ldr s0, 0x128"), + (0x124, 0x5c000040, "ldr d0, 0x12c"), + (0x128, 0x9cffffdf, "ldr q31, 0x120"), + (0x12c, 0xd8000040, "prfm pldl1keep, 0x134"), + (0x130, 0xd8ffffd5, "prfm pstl3strm, 0x128"), + (0x134, 0xd8000059, "prfm #0x19, 0x13c"), + (0x138, 0xd8000058, "prfm #0x18, 0x140"), + (0x13c, 0xf9800000, "prfm pldl1keep, [x0]"), + (0x140, 0xf98007e6, "prfm pldslckeep, [sp, #8]"), + (0x144, 0xf9bffc0b, "prfm plil2strm, [x0, #32760]"), + (0x148, 0xf8a16810, "prfm pstl1keep, [x0, x1]"), + (0x14c, 0xf8a17813, "prfm pstl2strm, [x0, x1, lsl #3]"), + (0x150, 0xf8a14804, "prfm pldl3keep, [x0, w1, uxtw]"), + (0x154, 0xf8a1d804, "prfm pldl3keep, [x0, w1, sxtw #3]"), + (0x158, 0xf980041f, "prfm #0x1f, [x0, #8]"), + (0x15c, 0xf980041f, "prfm #0x1f, [x0, #8]"), + (0x160, 0xf9800418, "prfm ir, [x0, #8]"), + (0x164, 0xf89f8000, "prfum pldl1keep, [x0, #-8]"), + (0x168, 0xf8801011, "prfum pstl1strm, [x0, #1]"), + (0x16c, 0xf89ff018, "prfum #0x18, [x0, #-1]"), + (0x170, 0xf88ff3ff, "prfum #0x1f, [sp, #255]"), + (0x174, 0x885f7c20, "ldxr w0, [x1]"), + (0x178, 0xc85f7fe0, "ldxr x0, [sp]"), + (0x17c, 0xc8027c20, "stxr w2, x0, [x1]"), + (0x180, 0x88027fe0, "stxr w2, w0, [sp]"), + (0x184, 0x085f7c20, "ldxrb w0, [x1]"), + (0x188, 0x08027c20, "stxrb w2, w0, [x1]"), + (0x18c, 0x485f7c20, "ldxrh w0, [x1]"), + (0x190, 0x48027c20, "stxrh w2, w0, [x1]"), + (0x194, 0xb8200041, "ldadd w0, w1, [x2]"), + (0x198, 0xf82003e1, "ldadd x0, x1, [sp]"), + (0x19c, 0x38200041, "ldaddb w0, w1, [x2]"), + (0x1a0, 0x78200041, "ldaddh w0, w1, [x2]"), + (0x1a4, 0xf8201041, "ldclr x0, x1, [x2]"), + (0x1a8, 0xb8202041, "ldeor w0, w1, [x2]"), + (0x1ac, 0xb8203041, "ldset w0, w1, [x2]"), + (0x1b0, 0xb8204041, "ldsmax w0, w1, [x2]"), + (0x1b4, 0xf8205041, "ldsmin x0, x1, [x2]"), + (0x1b8, 0xb8206041, "ldumax w0, w1, [x2]"), + (0x1bc, 0xf8207041, "ldumin x0, x1, [x2]"), + (0x1c0, 0x38204041, "ldsmaxb w0, w1, [x2]"), + (0x1c4, 0x78207041, "lduminh w0, w1, [x2]"), + (0x1c8, 0xb820005f, "stadd w0, [x2]"), + (0x1cc, 0xf820005f, "stadd x0, [x2]"), + (0x1d0, 0x3820105f, "stclrb w0, [x2]"), + (0x1d4, 0xb8208041, "swp w0, w1, [x2]"), + (0x1d8, 0xf820805f, "swp x0, xzr, [x2]"), + (0x1dc, 0x38208041, "swpb w0, w1, [x2]"), + (0x1e0, 0x783f8041, "swph wzr, w1, [x2]"), + (0x1e4, 0x88a07c41, "cas w0, w1, [x2]"), + (0x1e8, 0xc8a07fe1, "cas x0, x1, [sp]"), + (0x1ec, 0x08a07c41, "casb w0, w1, [x2]"), + (0x1f0, 0x48a07c5f, "cash w0, wzr, [x2]"), + (0x1f4, 0x08207c82, "casp w0, w1, w2, w3, [x4]"), + (0x1f8, 0x48207c82, "casp x0, x1, x2, x3, [x4]"), + (0x1fc, 0x483e7c82, "casp x30, xzr, x2, x3, [x4]"), + (0x200, 0x48227c9e, "casp x2, x3, x30, xzr, [x4]"), + (0x204, 0x887f0440, "ldxp w0, w1, [x2]"), + (0x208, 0xc87f07e0, "ldxp x0, x1, [sp]"), + (0x20c, 0x88200861, "stxp w0, w1, w2, [x3]"), + (0x210, 0xc8200861, "stxp w0, x1, x2, [x3]"), + (0x214, 0xc8207fff, "stxp w0, xzr, xzr, [sp]"), + (0x218, 0x2d400440, "ldp s0, s1, [x2]"), + (0x21c, 0x6d408440, "ldp d0, d1, [x2, #8]"), + (0x220, 0xadc08440, "ldp q0, q1, [x2, #16]!"), + (0x224, 0xacbf7c40, "stp q0, q31, [x2], #-32"), + (0x228, 0x2d2007e0, "stp s0, s1, [sp, #-256]"), + (0x22c, 0x6d1f805f, "stp d31, d0, [x2, #504]"), + (0x230, 0xad1f8440, "stp q0, q1, [x2, #1008]"), + (0x234, 0x3d400020, "ldr b0, [x1]"), + (0x238, 0x7d400420, "ldr h0, [x1, #2]"), + (0x23c, 0xbd400420, "ldr s0, [x1, #4]"), + (0x240, 0xfd400420, "ldr d0, [x1, #8]"), + (0x244, 0x3dc00420, "ldr q0, [x1, #16]"), + (0x248, 0x3dfffc20, "ldr q0, [x1, #65520]"), + (0x24c, 0x3c5ff020, "ldur b0, [x1, #-1]"), + (0x250, 0x7c401020, "ldur h0, [x1, #1]"), + (0x254, 0xbc402020, "ldur s0, [x1, #2]"), + (0x258, 0xfc5f8020, "ldur d0, [x1, #-8]"), + (0x25c, 0x3cdf0020, "ldur q0, [x1, #-16]"), + (0x260, 0x3cc08020, "ldur q0, [x1, #8]"), + (0x264, 0x3c9f0fe0, "str q0, [sp, #-16]!"), + (0x268, 0xfc008420, "str d0, [x1], #8"), + (0x26c, 0x3c001c20, "str b0, [x1, #1]!"), + (0x270, 0x3ca27820, "str q0, [x1, x2, lsl #4]"), + (0x274, 0x3ca26820, "str q0, [x1, x2]"), + (0x278, 0xfc22d820, "str d0, [x1, w2, sxtw #3]"), + (0x27c, 0xbc224820, "str s0, [x1, w2, uxtw]"), + (0x280, 0x7c227820, "str h0, [x1, x2, lsl #1]"), + (0x284, 0x3c226820, "str b0, [x1, x2]"), + (0x288, 0x3c9f0020, "stur q0, [x1, #-16]"), + (0x290, 0x4c40a000, "ld1 {v0.16b-v1.16b}, [x0]"), + (0x294, 0x4cdfa000, "ld1 {v0.16b-v1.16b}, [x0], #32"), + (0x298, 0x4cc1a000, "ld1 {v0.16b-v1.16b}, [x0], x1"), + (0x29c, 0x4cc17400, "ld1 {v0.8h}, [x0], x1"), + (0x2a0, 0x0cc17c00, "ld1 {v0.1d}, [x0], x1"), + (0x2a4, 0x0cdf63e0, "ld1 {v0.8b-v2.8b}, [sp], #24"), + (0x2a8, 0x4cdf2800, "ld1 {v0.4s-v3.4s}, [x0], #64"), + (0x2ac, 0x4c40281f, "ld1 {v31.4s-v2.4s}, [x0]"), + (0x2b0, 0x4cc32c1e, "ld1 {v30.2d-v1.2d}, [x0], x3"), + (0x2b4, 0x0c40a800, "ld1 {v0.2s-v1.2s}, [x0]"), + (0x2b8, 0x0cdf2400, "ld1 {v0.4h-v3.4h}, [x0], #32"), + (0x2bc, 0x4c408800, "ld2 {v0.4s-v1.4s}, [x0]"), + (0x2c0, 0x4cdf8000, "ld2 {v0.16b-v1.16b}, [x0], #32"), + (0x2c4, 0x0cc28000, "ld2 {v0.8b-v1.8b}, [x0], x2"), + (0x2c8, 0x4cdf4c00, "ld3 {v0.2d-v2.2d}, [x0], #48"), + (0x2cc, 0x4c40441f, "ld3 {v31.8h-v1.8h}, [x0]"), + (0x2d0, 0x4c400000, "ld4 {v0.16b-v3.16b}, [x0]"), + (0x2d4, 0x0cc5001e, "ld4 {v30.8b-v1.8b}, [x0], x5"), + (0x2d8, 0x0cdf07e0, "ld4 {v0.4h-v3.4h}, [sp], #32"), + (0x2dc, 0x4c00a000, "st1 {v0.16b-v1.16b}, [x0]"), + (0x2e0, 0x4c9f2800, "st1 {v0.4s-v3.4s}, [x0], #64"), + (0x2e4, 0x0c812000, "st1 {v0.8b-v3.8b}, [x0], x1"), + (0x2e8, 0x0c9f83e0, "st2 {v0.8b-v1.8b}, [sp], #16"), + (0x2ec, 0x4c008c00, "st2 {v0.2d-v1.2d}, [x0]"), + (0x2f0, 0x4c864800, "st3 {v0.4s-v2.4s}, [x0], x6"), + (0x2f4, 0x4c9f001d, "st4 {v29.16b-v0.16b}, [x0], #64"), + (0x2f8, 0x0d400000, "ld1 {v0.b}[0], [x0]"), + (0x2fc, 0x4d401c00, "ld1 {v0.b}[15], [x0]"), + (0x300, 0x0ddf1c00, "ld1 {v0.b}[7], [x0], #1"), + (0x304, 0x4d405800, "ld1 {v0.h}[7], [x0]"), + (0x308, 0x0ddf4000, "ld1 {v0.h}[0], [x0], #2"), + (0x30c, 0x4dc29000, "ld1 {v0.s}[3], [x0], x2"), + (0x310, 0x0ddf8000, "ld1 {v0.s}[0], [x0], #4"), + (0x314, 0x4dc28400, "ld1 {v0.d}[1], [x0], x2"), + (0x318, 0x0ddf841f, "ld1 {v31.d}[0], [x0], #8"), + (0x31c, 0x0d600c00, "ld2 {v0.b-v1.b}[3], [x0]"), + (0x320, 0x0dff5800, "ld2 {v0.h-v1.h}[3], [x0], #4"), + (0x324, 0x0de49000, "ld2 {v0.s-v1.s}[1], [x0], x4"), + (0x328, 0x4dff8400, "ld2 {v0.d-v1.d}[1], [x0], #16"), + (0x32c, 0x0ddfb000, "ld3 {v0.s-v2.s}[1], [x0], #12"), + (0x330, 0x4d40241f, "ld3 {v31.b-v1.b}[9], [x0]"), + (0x334, 0x0d60a400, "ld4 {v0.d-v3.d}[0], [x0]"), + (0x338, 0x0dff73e0, "ld4 {v0.h-v3.h}[2], [sp], #8"), + (0x33c, 0x4d008000, "st1 {v0.s}[2], [x0]"), + (0x340, 0x0d9f0400, "st1 {v0.b}[1], [x0], #1"), + (0x344, 0x4da38400, "st2 {v0.d-v1.d}[1], [x0], x3"), + (0x348, 0x4d9f6800, "st3 {v0.h-v2.h}[5], [x0], #6"), + (0x34c, 0x4d20b000, "st4 {v0.s-v3.s}[3], [x0]"), + (0x350, 0x4d40c000, "ld1r {v0.16b}, [x0]"), + (0x354, 0x0ddfc000, "ld1r {v0.8b}, [x0], #1"), + (0x358, 0x4ddfc800, "ld1r {v0.4s}, [x0], #4"), + (0x35c, 0x4dc1cc00, "ld1r {v0.2d}, [x0], x1"), + (0x360, 0x0ddfc400, "ld1r {v0.4h}, [x0], #2"), + (0x364, 0x4d60c800, "ld2r {v0.4s-v1.4s}, [x0]"), + (0x368, 0x4dffc400, "ld2r {v0.8h-v1.8h}, [x0], #4"), + (0x36c, 0x0de3cc00, "ld2r {v0.1d-v1.1d}, [x0], x3"), + (0x370, 0x0ddfe800, "ld3r {v0.2s-v2.2s}, [x0], #12"), + (0x374, 0x4d40ec1f, "ld3r {v31.2d-v1.2d}, [x0]"), + (0x378, 0x4d60ec00, "ld4r {v0.2d-v3.2d}, [x0]"), + (0x37c, 0x4dffe3e0, "ld4r {v0.16b-v3.16b}, [sp], #4"), + (0x380, 0x4de7e400, "ld4r {v0.8h-v3.8h}, [x0], x7"), + (0x384, 0x98000040, "ldrsw x0, 0x38c"), + (0x388, 0x98ffffc3, "ldrsw x3, 0x380"), + (0x38c, 0x9800005f, "ldrsw xzr, 0x394"), + ]; + + #[test] + fn matches_objdump() { + for &(pc, word, expected) in ROWS { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:#010x}: {e:?}")); + assert_eq!( + instruction.display_at(pc).to_string(), + expected, + "{word:#010x}" + ); + } + } + + /// `(word, text)`: GNU objdump 2.46's spelling of the words whose identity the decoded + /// form must keep (acquire/release, unscaled and unprivileged offsets, non-temporal + /// pairs, `ld1` arrangements, byte register offsets, `rprfm`). + const IDENTITY_ROWS: &[(u32, &str)] = &[ + (0x88dffc20, "ldar w0, [x1]"), + (0xc8dfffe0, "ldar x0, [sp]"), + (0x08dffc20, "ldarb w0, [x1]"), + (0x48dffc20, "ldarh w0, [x1]"), + (0x88df7c20, "ldlar w0, [x1]"), + (0xc8df7c20, "ldlar x0, [x1]"), + (0x08df7c20, "ldlarb w0, [x1]"), + (0x48df7c20, "ldlarh w0, [x1]"), + (0x889f7c20, "stllr w0, [x1]"), + (0xc89f7c20, "stllr x0, [x1]"), + (0x089f7c20, "stllrb w0, [x1]"), + (0x489f7c20, "stllrh w0, [x1]"), + (0x889ffc20, "stlr w0, [x1]"), + (0xc89fffe0, "stlr x0, [sp]"), + (0x089ffc20, "stlrb w0, [x1]"), + (0x489ffc20, "stlrh w0, [x1]"), + (0x889ffc1f, "stlr wzr, [x0]"), + (0xb8bfc020, "ldapr w0, [x1]"), + (0xf8bfc020, "ldapr x0, [x1]"), + (0x38bfc020, "ldaprb w0, [x1]"), + (0x78bfc020, "ldaprh w0, [x1]"), + (0x99404020, "ldapur w0, [x1, #4]"), + (0xd95f8020, "ldapur x0, [x1, #-8]"), + (0x19404020, "ldapurb w0, [x1, #4]"), + (0x59400020, "ldapurh w0, [x1]"), + (0x19c04020, "ldapursb w0, [x1, #4]"), + (0x19804020, "ldapursb x0, [x1, #4]"), + (0x59c04020, "ldapursh w0, [x1, #4]"), + (0x59804020, "ldapursh x0, [x1, #4]"), + (0x99804020, "ldapursw x0, [x1, #4]"), + (0x99004020, "stlur w0, [x1, #4]"), + (0xd9100020, "stlur x0, [x1, #-256]"), + (0x19000020, "stlurb w0, [x1]"), + (0x590ff020, "stlurh w0, [x1, #255]"), + (0x885ffc20, "ldaxr w0, [x1]"), + (0xc85fffe0, "ldaxr x0, [sp]"), + (0x085ffc20, "ldaxrb w0, [x1]"), + (0x485ffc20, "ldaxrh w0, [x1]"), + (0x8802fc20, "stlxr w2, w0, [x1]"), + (0xc802ffe0, "stlxr w2, x0, [sp]"), + (0x0802fc20, "stlxrb w2, w0, [x1]"), + (0x4802fc20, "stlxrh w2, w0, [x1]"), + (0x887f8440, "ldaxp w0, w1, [x2]"), + (0xc87f87e0, "ldaxp x0, x1, [sp]"), + (0x88238440, "stlxp w3, w0, w1, [x2]"), + (0xc8238440, "stlxp w3, x0, x1, [x2]"), + (0x88e07c41, "casa w0, w1, [x2]"), + (0x88a0fc41, "casl w0, w1, [x2]"), + (0xc8e0fc41, "casal x0, x1, [x2]"), + (0x08e07c41, "casab w0, w1, [x2]"), + (0x48e0fc41, "casalh w0, w1, [x2]"), + (0x08a0fc5f, "caslb w0, wzr, [x2]"), + (0x08607c82, "caspa w0, w1, w2, w3, [x4]"), + (0x4820fc82, "caspl x0, x1, x2, x3, [x4]"), + (0x4860fc82, "caspal x0, x1, x2, x3, [x4]"), + (0xb8a08041, "swpa w0, w1, [x2]"), + (0xf8608041, "swpl x0, x1, [x2]"), + (0xf8e0805f, "swpal x0, xzr, [x2]"), + (0x38a08041, "swpab w0, w1, [x2]"), + (0x78608041, "swplh w0, w1, [x2]"), + (0x38e08041, "swpalb w0, w1, [x2]"), + (0xb8a00041, "ldadda w0, w1, [x2]"), + (0xb8600041, "ldaddl w0, w1, [x2]"), + (0xf8e00041, "ldaddal x0, x1, [x2]"), + (0xb8a0003f, "ldadda w0, wzr, [x1]"), + (0xf8e0003f, "ldaddal x0, xzr, [x1]"), + (0xb860003f, "staddl w0, [x1]"), + (0x38a0003f, "ldaddab w0, wzr, [x1]"), + (0x7860003f, "staddlh w0, [x1]"), + (0x38e0003f, "ldaddalb w0, wzr, [x1]"), + (0x3860003f, "staddlb w0, [x1]"), + (0xb8a01041, "ldclra w0, w1, [x2]"), + (0xb8602041, "ldeorl w0, w1, [x2]"), + (0xb8a0305f, "ldseta w0, wzr, [x2]"), + (0xf8e04041, "ldsmaxal x0, x1, [x2]"), + (0xb8605041, "ldsminl w0, w1, [x2]"), + (0x78e06041, "ldumaxalh w0, w1, [x2]"), + (0x38a07041, "lduminab w0, w1, [x2]"), + (0xb8400020, "ldur w0, [x1]"), + (0xb8408020, "ldur w0, [x1, #8]"), + (0xf8410020, "ldur x0, [x1, #16]"), + (0xf85f8020, "ldur x0, [x1, #-8]"), + (0x38408020, "ldurb w0, [x1, #8]"), + (0x78404020, "ldurh w0, [x1, #4]"), + (0x38808020, "ldursb x0, [x1, #8]"), + (0x78c08020, "ldursh w0, [x1, #8]"), + (0xb8808020, "ldursw x0, [x1, #8]"), + (0xf8008020, "stur x0, [x1, #8]"), + (0x38001020, "sturb w0, [x1, #1]"), + (0xf9400420, "ldr x0, [x1, #8]"), + (0xf9400020, "ldr x0, [x1]"), + (0xb8404820, "ldtr w0, [x1, #4]"), + (0xf8400820, "ldtr x0, [x1]"), + (0x385ff820, "ldtrb w0, [x1, #-1]"), + (0x78402820, "ldtrh w0, [x1, #2]"), + (0x38c00820, "ldtrsb w0, [x1]"), + (0x78900820, "ldtrsh x0, [x1, #-256]"), + (0xb8804820, "ldtrsw x0, [x1, #4]"), + (0xf8008820, "sttr x0, [x1, #8]"), + (0x38000820, "sttrb w0, [x1]"), + (0x780ff820, "sttrh w0, [x1, #255]"), + (0xf9800020, "prfm pldl1keep, [x1]"), + (0xf8800020, "prfum pldl1keep, [x1]"), + (0xf8808020, "prfum pldl1keep, [x1, #8]"), + (0xf9800420, "prfm pldl1keep, [x1, #8]"), + (0xf89f8033, "prfum pstl2strm, [x1, #-8]"), + (0xf8a26820, "prfm pldl1keep, [x1, x2]"), + (0xf8a27820, "prfm pldl1keep, [x1, x2, lsl #3]"), + (0xf8a24820, "prfm pldl1keep, [x1, w2, uxtw]"), + (0xf8a2d820, "prfm pldl1keep, [x1, w2, sxtw #3]"), + (0xf8a2e820, "prfm pldl1keep, [x1, x2, sxtx]"), + (0xf8a24838, "rprfm pldkeep, x2, [x1]"), + (0xf8a24839, "rprfm pstkeep, x2, [x1]"), + (0xf8bf4bfc, "rprfm pldstrm, xzr, [sp]"), + (0xf8a2483d, "rprfm pststrm, x2, [x1]"), + (0xf8bf483c, "rprfm pldstrm, xzr, [x1]"), + (0xf8bf4bfd, "rprfm pststrm, xzr, [sp]"), + (0xf8a2483a, "rprfm #2, x2, [x1]"), + (0x38627820, "ldrb w0, [x1, x2, lsl #0]"), + (0x38626820, "ldrb w0, [x1, x2]"), + (0x38625820, "ldrb w0, [x1, w2, uxtw #0]"), + (0x3862d820, "ldrb w0, [x1, w2, sxtw #0]"), + (0x3862c820, "ldrb w0, [x1, w2, sxtw]"), + (0x3862f820, "ldrb w0, [x1, x2, sxtx #0]"), + (0x38a27820, "ldrsb x0, [x1, x2, lsl #0]"), + (0x38227820, "strb w0, [x1, x2, lsl #0]"), + (0x3c207962, "str b2, [x11, x0, lsl #0]"), + (0x3c206962, "str b2, [x11, x0]"), + (0x3c605962, "ldr b2, [x11, w0, uxtw #0]"), + (0x3ca07962, "str q2, [x11, x0, lsl #4]"), + (0x3ca06962, "str q2, [x11, x0]"), + (0x7c607962, "ldr h2, [x11, x0, lsl #1]"), + (0xf8607962, "ldr x2, [x11, x0, lsl #3]"), + (0xa8400440, "ldnp x0, x1, [x2]"), + (0x28600440, "ldnp w0, w1, [x2, #-256]"), + (0xa8008440, "stnp x0, x1, [x2, #8]"), + (0x28010440, "stnp w0, w1, [x2, #8]"), + (0x2c400440, "ldnp s0, s1, [x2]"), + (0x6c408440, "ldnp d0, d1, [x2, #8]"), + (0xac408440, "ldnp q0, q1, [x2, #16]"), + (0x2c000440, "stnp s0, s1, [x2]"), + (0x6c3f8440, "stnp d0, d1, [x2, #-8]"), + (0xac3f0440, "stnp q0, q1, [x2, #-32]"), + (0x4c407020, "ld1 {v0.16b}, [x1]"), + (0x0c407020, "ld1 {v0.8b}, [x1]"), + (0x4c407420, "ld1 {v0.8h}, [x1]"), + (0x0c407420, "ld1 {v0.4h}, [x1]"), + (0x4c407820, "ld1 {v0.4s}, [x1]"), + (0x0c407820, "ld1 {v0.2s}, [x1]"), + (0x4c407c20, "ld1 {v0.2d}, [x1]"), + (0x0c407c20, "ld1 {v0.1d}, [x1]"), + (0x4c4073ff, "ld1 {v31.16b}, [sp]"), + (0x4cdf7020, "ld1 {v0.16b}, [x1], #16"), + (0x0cdf7020, "ld1 {v0.8b}, [x1], #8"), + (0x0cdf7c20, "ld1 {v0.1d}, [x1], #8"), + (0x4cdf7820, "ld1 {v0.4s}, [x1], #16"), + (0x0cc27420, "ld1 {v0.4h}, [x1], x2"), + (0x4cc27020, "ld1 {v0.16b}, [x1], x2"), + (0x4c007020, "st1 {v0.16b}, [x1]"), + (0x4c007420, "st1 {v0.8h}, [x1]"), + (0x0c9f7820, "st1 {v0.2s}, [x1], #8"), + (0x4c9f73ff, "st1 {v31.16b}, [sp], #16"), + (0x0c007c20, "st1 {v0.1d}, [x1]"), + (0x3dc00020, "ldr q0, [x1]"), + (0xfd400020, "ldr d0, [x1]"), + (0x3cc10020, "ldur q0, [x1, #16]"), + (0x3cc08020, "ldur q0, [x1, #8]"), + (0xfc5f8020, "ldur d0, [x1, #-8]"), + (0xfc400020, "ldur d0, [x1]"), + (0xbc404020, "ldur s0, [x1, #4]"), + (0x7c402020, "ldur h0, [x1, #2]"), + (0x3c400020, "ldur b0, [x1]"), + (0x3c401020, "ldur b0, [x1, #1]"), + (0x3d400420, "ldr b0, [x1, #1]"), + (0x3c810020, "stur q0, [x1, #16]"), + (0xfc1f8020, "stur d0, [x1, #-8]"), + (0xfd400420, "ldr d0, [x1, #8]"), + ]; + + #[test] + fn keeps_instruction_identity() { + for &(word, expected) in IDENTITY_ROWS { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:#010x}: {e:?}")); + assert_eq!(instruction.display().to_string(), expected, "{word:#010x}"); + } + } + + #[test] + fn without_pc_targets_are_relative() { + assert_eq!( + decode(0x18000040).unwrap().display().to_string(), + "ldr w0, .+8" + ); + assert_eq!( + decode(0xd8ffffd5).unwrap().display().to_string(), + "prfm pstl3strm, .-8" + ); + } +} diff --git a/src/isa/aarch64/src/format/mod.rs b/src/isa/aarch64/src/format/mod.rs new file mode 100644 index 0000000..71e6bc9 --- /dev/null +++ b/src/isa/aarch64/src/format/mod.rs @@ -0,0 +1,211 @@ +//! Textual disassembly of decoded instructions, in GNU binutils (`objdump`) spelling. +//! +//! The text is a presentation of an already decoded [`Instruction`]: nothing here +//! decodes bits. Assembly aliases (`mov`, `cmp`, `ret`, `lsl #n`, ...) are chosen +//! here, from the decoded fields, the way the Arm ARM's alias conditions describe. +//! +//! * [`Instruction::display`] renders without an address: PC-relative operands +//! print as `.+N` / `.-N`. +//! * [`Instruction::display_at`] renders at `pc`: targets print as absolute +//! addresses (`0x1008`). +//! * [`Instruction::display_with`] additionally annotates targets with the symbol +//! that contains them (`0x1008 `), through [`Symbols`]. +//! * [`word`], [`word_at`] and [`word_with`] start from a raw word and render +//! `.word 0x...` for one the decoder does not recognise. + +use crate::decode::{Instruction, decode}; +use core::fmt::{self, Display, Formatter}; + +mod fp; +mod integer; +mod memory; +mod reg; +mod system; +mod vector; + +/// Resolves a code or data address to the symbol that contains it. +pub trait Symbols { + /// The symbol containing `addr` and the byte offset of `addr` into it. + fn lookup(&self, addr: u64) -> Option<(&str, u64)>; +} + +/// No symbols: targets print bare. +impl Symbols for () { + fn lookup(&self, _: u64) -> Option<(&str, u64)> { + None + } +} + +/// Where an instruction is, and what is known about its surroundings. +#[derive(Clone, Copy)] +pub(crate) struct Context<'a> { + /// The instruction's address, when known. Without it a PC-relative operand + /// prints as an offset from `.`. + pub pc: Option, + pub symbols: &'a dyn Symbols, +} + +impl Context<'_> { + /// A PC-relative code or data target `offset` bytes from the instruction: + /// `.+N` / `.-N` without a pc, else `0x` followed by ` ` or + /// ` ` when a symbol contains it. Address arithmetic wraps; a + /// disassembler must never panic on the bytes it is given. + pub fn target(&self, f: &mut Formatter<'_>, offset: i64) -> fmt::Result { + match self.pc { + None => { + let sign = if offset < 0 { '-' } else { '+' }; + write!(f, ".{sign}{}", offset.unsigned_abs()) + } + Some(pc) => self.address(f, pc.wrapping_add(offset as u64)), + } + } + + /// Like [`Self::target`] for `adrp`: `offset` is relative to the 4 KiB page + /// containing the instruction. + pub fn page_target(&self, f: &mut Formatter<'_>, offset: i64) -> fmt::Result { + match self.pc { + None => { + let sign = if offset < 0 { '-' } else { '+' }; + write!(f, ".{sign}{}", offset.unsigned_abs()) + } + Some(pc) => self.address(f, (pc & !0xfff).wrapping_add(offset as u64)), + } + } + + /// An absolute address with its symbol annotation. + fn address(&self, f: &mut Formatter<'_>, addr: u64) -> fmt::Result { + write!(f, "{addr:#x}")?; + match self.symbols.lookup(addr) { + Some((name, 0)) => write!(f, " <{name}>"), + Some((name, offset)) => write!(f, " <{name}+{offset:#x}>"), + None => Ok(()), + } + } +} + +/// An [`Instruction`] ready to print; see [`Instruction::display`]. +pub struct Disassembly<'a> { + instruction: &'a Instruction, + cx: Context<'a>, +} + +impl Instruction { + /// Render without an address. + pub fn display(&self) -> Disassembly<'_> { + Disassembly { + instruction: self, + cx: Context { + pc: None, + symbols: &(), + }, + } + } + + /// Render at address `pc`. + pub fn display_at(&self, pc: u64) -> Disassembly<'_> { + Disassembly { + instruction: self, + cx: Context { + pc: Some(pc), + symbols: &(), + }, + } + } + + /// Render at address `pc`, annotating targets from `symbols`. + pub fn display_with<'a>(&'a self, pc: u64, symbols: &'a dyn Symbols) -> Disassembly<'a> { + Disassembly { + instruction: self, + cx: Context { + pc: Some(pc), + symbols, + }, + } + } +} + +impl Display for Disassembly<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write(f, &self.cx, self.instruction) + } +} + +/// A raw instruction word ready to print; see [`word`]. +pub struct Word<'a> { + word: u32, + cx: Context<'a>, +} + +/// Render `word` without an address; `.word 0x...` when it does not decode. +pub fn word(word: u32) -> Word<'static> { + Word { + word, + cx: Context { + pc: None, + symbols: &(), + }, + } +} + +/// Render `word` at `pc`; `.word 0x...` when it does not decode. +pub fn word_at(word: u32, pc: u64) -> Word<'static> { + Word { + word, + cx: Context { + pc: Some(pc), + symbols: &(), + }, + } +} + +/// Render `word` at `pc`, annotating targets from `symbols`. +pub fn word_with(word: u32, pc: u64, symbols: &dyn Symbols) -> Word<'_> { + Word { + word, + cx: Context { + pc: Some(pc), + symbols, + }, + } +} + +impl Display for Word<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match decode(self.word) { + Ok(instruction) => write(f, &self.cx, &instruction), + Err(_) => write!(f, ".word {:#010x}", self.word), + } + } +} + +/// Route `instruction` to the module that owns its family. Every variant is named +/// here, so a new one does not compile until it has a formatter. +fn write(f: &mut Formatter<'_>, cx: &Context<'_>, instruction: &Instruction) -> fmt::Result { + use Instruction::*; + match instruction { + Movz(_) | Movn(_) | Movk(_) | ArithImm(_) | ArithReg(_) | ArithExt(_) | LogicReg(_) + | LogicImm(_) | Mul(_) | MulLong(_) | MulHigh(_) | Adr(_) | Csel(_) | Variable(_) + | CondCompare(_) | ArithCarry(_) | Unary(_) | Bitfield(_) | Extract(_) => { + integer::write(f, cx, instruction) + } + Memory(_) | Pair(_) | Literal(_) | SimdLiteral(_) | Exclusive(_) | Atomic(_) + | Prefetch(_) | PrefetchLiteral(_) | RangePrefetch(_) | SimdPair(_) | SimdMemory(_) + | SimdStruct(_) => memory::write(f, cx, instruction), + TestBranch(_) | Call(_) | Indirect(_) | B(_) | BCond(_) | Eret | System(_) | DcZva(_) + | AddressTranslate(_) | CacheOp(_) | Tlbi(_) | Clrex(_) | Nop | Hint(_) | Dsb(_) + | Dmb(_) | Isb(_) | Sev | Sevl | Wfe | Wfi | Yield | Brk(_) | Svc(_) | Psci | Pauth(_) + | Gxf(_) => system::write(f, cx, instruction), + SimdImm(_) + | SimdCompareZero(_) + | SimdDup { .. } + | SimdDupElement(_) + | SimdInsert(_) + | SimdInsertElement(_) + | SimdExtract(_) + | SimdExt(_) + | SimdAlu(_) + | SimdTable(_) + | Simd(_) => vector::write(f, cx, instruction), + Fp(_) | SimdFmov(_) => fp::write(f, cx, instruction), + } +} diff --git a/src/isa/aarch64/src/format/reg.rs b/src/isa/aarch64/src/format/reg.rs new file mode 100644 index 0000000..ece25a6 --- /dev/null +++ b/src/isa/aarch64/src/format/reg.rs @@ -0,0 +1,76 @@ +//! Register and condition spelling shared by every family. + +use crate::decode::{Condition, Width}; +use core::fmt::{self, Formatter}; + +/// A general register where encoding 31 is the zero register: `x0`..`x30`, `xzr`, +/// `w0`..`w30`, `wzr`. +pub fn gpr(f: &mut Formatter<'_>, width: Width, r: u8) -> fmt::Result { + let prefix = match width { + Width::W32 => 'w', + Width::X64 => 'x', + }; + if r == 31 { + write!(f, "{prefix}zr") + } else { + write!(f, "{prefix}{r}") + } +} + +/// A general register where encoding 31 is the stack pointer: `sp` / `wsp`. +pub fn gpr_sp(f: &mut Formatter<'_>, width: Width, r: u8) -> fmt::Result { + match (r, width) { + (31, Width::X64) => f.write_str("sp"), + (31, Width::W32) => f.write_str("wsp"), + _ => gpr(f, width, r), + } +} + +/// The condition code as binutils spells it (`cs`/`cc`, not `hs`/`lo`). +pub fn condition(c: Condition) -> &'static str { + match c { + Condition::Eq => "eq", + Condition::Ne => "ne", + Condition::Cs => "cs", + Condition::Cc => "cc", + Condition::Mi => "mi", + Condition::Pl => "pl", + Condition::Vs => "vs", + Condition::Vc => "vc", + Condition::Hi => "hi", + Condition::Ls => "ls", + Condition::Ge => "ge", + Condition::Lt => "lt", + Condition::Gt => "gt", + Condition::Le => "le", + Condition::Al => "al", + Condition::Nv => "nv", + } +} + +/// A SIMD&FP scalar register `bytes` wide: `b`, `h`, `s`, `d` or `q` for 1, 2, 4, 8, 16. +pub fn fp_scalar(f: &mut Formatter<'_>, bytes: u8, r: u8) -> fmt::Result { + let prefix = match bytes { + 1 => 'b', + 2 => 'h', + 4 => 's', + 8 => 'd', + _ => 'q', + }; + write!(f, "{prefix}{r}") +} + +/// The element suffix of a vector arrangement: `size` is log2 of the element width +/// in bytes (0 = bytes .. 3 = doublewords), `q` selects 128 bits over 64. +/// `(0, true)` is `16b`, `(2, false)` is `2s`, `(3, false)` is `1d`. +pub fn arrangement(f: &mut Formatter<'_>, size: u8, q: bool) -> fmt::Result { + let lanes = (if q { 16 } else { 8 }) >> size; + let element = ['b', 'h', 's', 'd'][usize::from(size & 3)]; + write!(f, "{lanes}{element}") +} + +/// A vector register with its arrangement: `v0.16b`. +pub fn vector(f: &mut Formatter<'_>, r: u8, size: u8, q: bool) -> fmt::Result { + write!(f, "v{r}.")?; + arrangement(f, size, q) +} diff --git a/src/isa/aarch64/src/format/system.rs b/src/isa/aarch64/src/format/system.rs new file mode 100644 index 0000000..3467381 --- /dev/null +++ b/src/isa/aarch64/src/format/system.rs @@ -0,0 +1,1259 @@ +//! Branches, system instructions and the families that are not data processing: +//! pointer authentication and guarded execution. +//! +//! Owns `TestBranch`, `Call`, `Indirect`, `B`, `BCond`, `Eret`, `System` (`mrs`/`msr`, +//! with every `SystemRegister` name), `DcZva`, `AddressTranslate`, `CacheOp`, `Tlbi`, +//! `Clrex`, `Nop`, `Hint`, `Dsb`, `Dmb`, `Isb`, `Sev`, `Sevl`, `Wfe`, `Wfi`, `Yield`, +//! `Brk`, `Svc`, `Psci`, `Pauth` and `Gxf`. + +use super::{ + Context, + reg::{condition, gpr}, +}; +use crate::decode::{self, IndirectKind, Instruction, SysOp, SystemRegister, Width}; +use crate::gxf; +use crate::pauth::{self, KeyId, Modifier}; +use core::fmt::{self, Formatter}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + use Instruction::*; + match *instruction { + TestBranch(test) => test_branch(f, cx, &test), + Call(call) => { + f.write_str(if call.link { "bl " } else { "b " })?; + cx.target(f, call.target) + } + B(offset) => { + f.write_str("b ")?; + cx.target(f, offset) + } + BCond(branch) => { + write!(f, "b.{} ", condition(branch.cond))?; + cx.target(f, branch.offset) + } + Indirect(indirect) => match indirect.kind { + IndirectKind::Jump => { + f.write_str("br ")?; + gpr(f, Width::X64, indirect.rn) + } + IndirectKind::Call => { + f.write_str("blr ")?; + gpr(f, Width::X64, indirect.rn) + } + IndirectKind::Return if indirect.rn == 30 => f.write_str("ret"), + IndirectKind::Return => { + f.write_str("ret ")?; + gpr(f, Width::X64, indirect.rn) + } + }, + Eret => f.write_str("eret"), + System(system) => system_register_move(f, &system), + DcZva(rt) => { + f.write_str("dc zva, ")?; + gpr(f, Width::X64, rt) + } + AddressTranslate(at) => { + write!( + f, + "at s1e{}{}, ", + if at.user { '0' } else { '1' }, + if at.write { 'w' } else { 'r' } + )?; + gpr(f, Width::X64, at.rt) + } + CacheOp(op) => cache_op(f, &op), + Tlbi(op) => tlbi(f, &op), + Clrex(option) => option_or_default(f, "clrex", option), + Nop => f.write_str("nop"), + Hint(n) => hint(f, n), + Dsb(option) => match option { + 0 => f.write_str("ssbb"), + 4 => f.write_str("pssbb"), + _ => barrier(f, "dsb", option), + }, + Dmb(option) => barrier(f, "dmb", option), + Isb(option) => option_or_default(f, "isb", option), + Sev => f.write_str("sev"), + Sevl => f.write_str("sevl"), + Wfe => f.write_str("wfe"), + Wfi => f.write_str("wfi"), + Yield => f.write_str("yield"), + Brk(imm) => write!(f, "brk #{imm:#x}"), + Svc(imm) => write!(f, "svc #{imm:#x}"), + Psci => f.write_str("hvc #0x0"), + Pauth(op) => pauth_op(f, op), + Gxf(op) => match op { + gxf::Op::Gexit => f.write_str(".inst 0x00201400"), + gxf::Op::Genter { imm5 } => write!(f, ".inst {:#010x}", 0x0020_1420 | u32::from(imm5)), + }, + _ => unreachable!("{instruction:?} is not a system instruction"), + } +} + +fn test_branch(f: &mut Formatter<'_>, cx: &Context<'_>, test: &decode::Test) -> fmt::Result { + match test.bit_index { + None => f.write_str(if test.negate { "cbnz " } else { "cbz " })?, + Some(_) => f.write_str(if test.negate { "tbnz " } else { "tbz " })?, + } + gpr(f, test.width, test.rt)?; + f.write_str(", ")?; + if let Some(bit) = test.bit_index { + write!(f, "#{bit}, ")?; + } + cx.target(f, test.offset) +} + +/// `mnemonic` alone for the architectural default option 15, else `mnemonic #imm`. +fn option_or_default(f: &mut Formatter<'_>, mnemonic: &str, option: u8) -> fmt::Result { + if option == 15 { + f.write_str(mnemonic) + } else { + write!(f, "{mnemonic} #{option:#x}") + } +} + +/// `dsb`/`dmb` with the barrier option named when binutils names it. +fn barrier(f: &mut Formatter<'_>, mnemonic: &str, option: u8) -> fmt::Result { + let name = match option { + 1 => "oshld", + 2 => "oshst", + 3 => "osh", + 5 => "nshld", + 6 => "nshst", + 7 => "nsh", + 9 => "ishld", + 10 => "ishst", + 11 => "ish", + 13 => "ld", + 14 => "st", + 15 => "sy", + _ => return write!(f, "{mnemonic} #{option:#04x}"), + }; + write!(f, "{mnemonic} {name}") +} + +fn hint(f: &mut Formatter<'_>, n: u8) -> fmt::Result { + let text = match n { + 0 => "nop", + 1 => "yield", + 2 => "wfe", + 3 => "wfi", + 4 => "sev", + 5 => "sevl", + 6 => "dgh", + // Hints 7, 8, 10, 12, 14 and 24 to 31 (`xpaclri`, `pacia1716`, `paciasp`, ...) are + // pointer authentication: the decoder yields `Pauth`, never `Hint`, for them. + 16 => "esb", + 17 => "psb csync", + 18 => "tsb csync", + 19 => "gcsb dsync", + 20 => "csdb", + 22 => "clrbhb", + 32 => "bti r", + 34 => "bti c", + 36 => "bti j", + 38 => "bti jc", + 40 => "chkfeat x16", + 48 => "stshh keep", + 49 => "stshh strm", + 50 => "shuh", + 51 => "shuh ph", + 52 => "stcph", + _ => return write!(f, "hint #{n:#x}"), + }; + f.write_str(text) +} + +/// How a named `sys` alias spells its `Xt` operand. +#[derive(Clone, Copy)] +enum Xt { + /// Never printed. + Never, + /// Always printed (`xzr` for 31). + Always, + /// Printed unless it is 31. + UnlessZero, +} + +const IC: &[(u8, u8, u8, &str, Xt)] = &[ + (0, 1, 0, "ialluis", Xt::Never), + (0, 5, 0, "iallu", Xt::Never), + (3, 5, 1, "ivau", Xt::Always), +]; + +const DC: &[(u8, u8, u8, &str, Xt)] = &[ + (0, 6, 1, "ivac", Xt::Always), + (0, 6, 2, "isw", Xt::Always), + (0, 6, 3, "igvac", Xt::Always), + (0, 6, 4, "igsw", Xt::Always), + (0, 6, 5, "igdvac", Xt::Always), + (0, 6, 6, "igdsw", Xt::Always), + (0, 10, 2, "csw", Xt::Always), + (0, 10, 4, "cgsw", Xt::Always), + (0, 10, 6, "cgdsw", Xt::Always), + (0, 14, 2, "cisw", Xt::Always), + (0, 14, 4, "cigsw", Xt::Always), + (0, 14, 6, "cigdsw", Xt::Always), + (3, 10, 1, "cvac", Xt::Always), + (3, 10, 3, "cgvac", Xt::Always), + (3, 10, 5, "cgdvac", Xt::Always), + (3, 11, 0, "cvaoc", Xt::Always), + (3, 11, 1, "cvau", Xt::Always), + (3, 11, 7, "cgdvaoc", Xt::Always), + (3, 14, 1, "civac", Xt::Always), + (3, 14, 3, "cigvac", Xt::Always), + (3, 14, 5, "cigdvac", Xt::Always), + (4, 14, 0, "cipae", Xt::Always), + (4, 14, 7, "cigdpae", Xt::Always), + (6, 14, 1, "cipapa", Xt::Always), + (6, 14, 5, "cigdpapa", Xt::Always), +]; + +const TLBI: &[(u8, u8, u8, &str, Xt)] = &[ + (0, 1, 0, "vmalle1os", Xt::UnlessZero), + (0, 1, 1, "vae1os", Xt::Always), + (0, 1, 2, "aside1os", Xt::Always), + (0, 1, 3, "vaae1os", Xt::Always), + (0, 1, 5, "vale1os", Xt::Always), + (0, 1, 7, "vaale1os", Xt::Always), + (0, 2, 1, "rvae1is", Xt::Always), + (0, 2, 3, "rvaae1is", Xt::Always), + (0, 2, 5, "rvale1is", Xt::Always), + (0, 2, 7, "rvaale1is", Xt::Always), + (0, 3, 0, "vmalle1is", Xt::UnlessZero), + (0, 3, 1, "vae1is", Xt::Always), + (0, 3, 2, "aside1is", Xt::Always), + (0, 3, 3, "vaae1is", Xt::Always), + (0, 3, 5, "vale1is", Xt::Always), + (0, 3, 7, "vaale1is", Xt::Always), + (0, 5, 1, "rvae1os", Xt::Always), + (0, 5, 3, "rvaae1os", Xt::Always), + (0, 5, 5, "rvale1os", Xt::Always), + (0, 5, 7, "rvaale1os", Xt::Always), + (0, 6, 1, "rvae1", Xt::Always), + (0, 6, 3, "rvaae1", Xt::Always), + (0, 6, 5, "rvale1", Xt::Always), + (0, 6, 7, "rvaale1", Xt::Always), + (0, 7, 0, "vmalle1", Xt::Never), + (0, 7, 1, "vae1", Xt::Always), + (0, 7, 2, "aside1", Xt::Always), + (0, 7, 3, "vaae1", Xt::Always), + (0, 7, 5, "vale1", Xt::Always), + (0, 7, 7, "vaale1", Xt::Always), + (4, 0, 1, "ipas2e1is", Xt::Always), + (4, 0, 2, "ripas2e1is", Xt::Always), + (4, 0, 5, "ipas2le1is", Xt::Always), + (4, 0, 6, "ripas2le1is", Xt::Always), + (4, 1, 0, "alle2os", Xt::UnlessZero), + (4, 1, 1, "vae2os", Xt::Always), + (4, 1, 4, "alle1os", Xt::UnlessZero), + (4, 1, 5, "vale2os", Xt::Always), + (4, 1, 6, "vmalls12e1os", Xt::UnlessZero), + (4, 2, 1, "rvae2is", Xt::Always), + (4, 2, 2, "vmallws2e1is", Xt::UnlessZero), + (4, 2, 5, "rvale2is", Xt::Always), + (4, 3, 0, "alle2is", Xt::UnlessZero), + (4, 3, 1, "vae2is", Xt::Always), + (4, 3, 4, "alle1is", Xt::UnlessZero), + (4, 3, 5, "vale2is", Xt::Always), + (4, 3, 6, "vmalls12e1is", Xt::UnlessZero), + (4, 4, 0, "ipas2e1os", Xt::Always), + (4, 4, 1, "ipas2e1", Xt::Always), + (4, 4, 2, "ripas2e1", Xt::Always), + (4, 4, 3, "ripas2e1os", Xt::Always), + (4, 4, 4, "ipas2le1os", Xt::Always), + (4, 4, 5, "ipas2le1", Xt::Always), + (4, 4, 6, "ripas2le1", Xt::Always), + (4, 4, 7, "ripas2le1os", Xt::Always), + (4, 5, 1, "rvae2os", Xt::Always), + (4, 5, 2, "vmallws2e1os", Xt::UnlessZero), + (4, 5, 5, "rvale2os", Xt::Always), + (4, 6, 1, "rvae2", Xt::Always), + (4, 6, 3, "vmallws2e1", Xt::Never), + (4, 6, 5, "rvale2", Xt::Always), + (4, 7, 0, "alle2", Xt::Never), + (4, 7, 1, "vae2", Xt::Always), + (4, 7, 4, "alle1", Xt::Never), + (4, 7, 5, "vale2", Xt::Always), + (4, 7, 6, "vmalls12e1", Xt::Never), + (6, 1, 0, "alle3os", Xt::Never), + (6, 1, 1, "vae3os", Xt::Always), + (6, 1, 4, "paallos", Xt::Never), + (6, 1, 5, "vale3os", Xt::Always), + (6, 2, 1, "rvae3is", Xt::Always), + (6, 2, 5, "rvale3is", Xt::Always), + (6, 3, 0, "alle3is", Xt::Never), + (6, 3, 1, "vae3is", Xt::Always), + (6, 3, 5, "vale3is", Xt::Always), + (6, 4, 3, "rpaos", Xt::Always), + (6, 4, 7, "rpalos", Xt::Always), + (6, 5, 1, "rvae3os", Xt::Always), + (6, 5, 5, "rvale3os", Xt::Always), + (6, 6, 1, "rvae3", Xt::Always), + (6, 6, 5, "rvale3", Xt::Always), + (6, 7, 0, "alle3", Xt::Never), + (6, 7, 1, "vae3", Xt::Always), + (6, 7, 4, "paall", Xt::Never), + (6, 7, 5, "vale3", Xt::Always), +]; + +fn lookup(table: &[(u8, u8, u8, &'static str, Xt)], op: &SysOp) -> Option<(&'static str, Xt)> { + table + .iter() + .find(|&&(op1, crm, op2, _, _)| (op1, crm, op2) == (op.op1, op.crm, op.op2)) + .map(|&(_, _, _, name, xt)| (name, xt)) +} + +/// `mnemonic name{, xt}` for a named alias, or the generic `sys` form. +fn named_sys( + f: &mut Formatter<'_>, + mnemonic: &str, + named: Option<(&str, Xt)>, + op: &SysOp, +) -> fmt::Result { + let Some((name, xt)) = named else { + return generic_sys(f, op); + }; + write!(f, "{mnemonic} {name}")?; + match xt { + Xt::Never => Ok(()), + Xt::UnlessZero if op.rt == 31 => Ok(()), + Xt::Always | Xt::UnlessZero => { + f.write_str(", ")?; + gpr(f, Width::X64, op.rt) + } + } +} + +fn generic_sys(f: &mut Formatter<'_>, op: &SysOp) -> fmt::Result { + write!(f, "sys #{}, C{}, C{}, #{}", op.op1, op.crn, op.crm, op.op2)?; + if op.rt != 31 { + f.write_str(", ")?; + gpr(f, Width::X64, op.rt)?; + } + Ok(()) +} + +fn cache_op(f: &mut Formatter<'_>, op: &SysOp) -> fmt::Result { + match lookup(IC, op) { + Some(ic) => named_sys(f, "ic", Some(ic), op), + None => named_sys(f, "dc", lookup(DC, op), op), + } +} + +fn tlbi(f: &mut Formatter<'_>, op: &SysOp) -> fmt::Result { + named_sys(f, "tlbi", lookup(TLBI, op), op) +} + +/// `(op1, crm, op2)` of the Apple IMP-DEF register in `System::apple[slot]` +/// (`S3__C15_C_`). +const APPLE: [(u8, u8, u8); 21] = [ + (1, 0, 0), + (1, 1, 0), + (1, 2, 0), + (1, 3, 0), + (1, 4, 0), + (1, 5, 0), + (1, 6, 0), + (2, 0, 0), + (2, 1, 0), + (2, 2, 0), + (2, 3, 0), + (2, 4, 0), + (2, 5, 0), + (2, 6, 0), + (2, 7, 0), + (2, 9, 0), + (2, 10, 0), + (5, 0, 0), + (5, 0, 1), + (5, 3, 1), + (7, 0, 4), +]; + +fn generic_register( + f: &mut Formatter<'_>, + op0: u8, + op1: u8, + crn: u8, + crm: u8, + op2: u8, +) -> fmt::Result { + write!(f, "s{op0}_{op1}_c{crn}_c{crm}_{op2}") +} + +fn register_name(f: &mut Formatter<'_>, register: SystemRegister) -> fmt::Result { + use SystemRegister as R; + let name = match register { + R::SpEl0 => "sp_el0", + R::SpEl1 => "sp_el1", + R::SpEl2 => "sp_el2", + R::SctlrEl2 => "sctlr_el2", + R::TcrEl2 => "tcr_el2", + R::Ttbr0El2 => "ttbr0_el2", + R::Ttbr1El2 => "ttbr1_el2", + R::MairEl2 => "mair_el2", + R::VbarEl2 => "vbar_el2", + R::ElrEl2 => "elr_el2", + R::SpsrEl2 => "spsr_el2", + R::EsrEl2 => "esr_el2", + R::FarEl2 => "far_el2", + R::HcrEl2 => "hcr_el2", + R::TpidrEl2 => "tpidr_el2", + R::CntvoffEl2 => "cntvoff_el2", + R::CurrentEl => "currentel", + R::SpsrEl1 => "spsr_el1", + R::ElrEl1 => "elr_el1", + R::EsrEl1 => "esr_el1", + R::FarEl1 => "far_el1", + R::ParEl1 => "par_el1", + R::VbarEl1 => "vbar_el1", + R::CpacrEl1 => "cpacr_el1", + R::SctlrEl1 => "sctlr_el1", + R::Ttbr0El1 => "ttbr0_el1", + R::Ttbr1El1 => "ttbr1_el1", + R::TcrEl1 => "tcr_el1", + R::MairEl1 => "mair_el1", + R::TpidrEl0 => "tpidr_el0", + R::TpidrroEl0 => "tpidrro_el0", + R::CntvctEl0 => "cntvct_el0", + R::CntvCtlEl0 => "cntv_ctl_el0", + R::CntvCvalEl0 => "cntv_cval_el0", + R::CntvTvalEl0 => "cntv_tval_el0", + R::TpidrEl1 => "tpidr_el1", + R::ContextidrEl1 => "contextidr_el1", + R::MdscrEl1 => "mdscr_el1", + R::OslarEl1 => "oslar_el1", + R::OsdlrEl1 => "osdlr_el1", + R::OslsrEl1 => "oslsr_el1", + R::CntkctlEl1 => "cntkctl_el1", + R::Fpcr => "fpcr", + R::Fpsr => "fpsr", + R::Daif => "daif", + R::SpSel => "spsel", + R::Pan => "pan", + R::IsrEl1 => "isr_el1", + R::IccSreEl1 => "icc_sre_el1", + R::IccPmrEl1 => "icc_pmr_el1", + R::IccBpr0El1 => "icc_bpr0_el1", + R::IccCtlrEl1 => "icc_ctlr_el1", + R::IccIgrpen0El1 => "icc_igrpen0_el1", + R::IccIgrpen1El1 => "icc_igrpen1_el1", + R::IccBpr1El1 => "icc_bpr1_el1", + R::CntpCtlEl0 => "cntp_ctl_el0", + R::CntpctEl0 => "cntpct_el0", + R::CntpTvalEl0 => "cntp_tval_el0", + R::CntpCvalEl0 => "cntp_cval_el0", + R::IccIar1El1 => "icc_iar1_el1", + R::IccEoir1El1 => "icc_eoir1_el1", + R::ActlrEl1 => "actlr_el1", + R::MidrEl1 => "midr_el1", + R::MpidrEl1 => "mpidr_el1", + R::RevidrEl1 => "revidr_el1", + R::CtrEl0 => "ctr_el0", + R::DczidEl0 => "dczid_el0", + R::CntfrqEl0 => "cntfrq_el0", + R::ClidrEl1 => "clidr_el1", + R::CcsidrEl1 => "ccsidr_el1", + R::CsselrEl1 => "csselr_el1", + R::IdAa64pfr0El1 => "id_aa64pfr0_el1", + R::IdAa64pfr1El1 => "id_aa64pfr1_el1", + R::IdAa64pfr2El1 => "id_aa64pfr2_el1", + R::IdAa64zfr0El1 => "id_aa64zfr0_el1", + R::IdAa64smfr0El1 => "id_aa64smfr0_el1", + R::IdAa64fpfr0El1 => "id_aa64fpfr0_el1", + R::IdAa64isar3El1 => "id_aa64isar3_el1", + R::AidrEl1 => "aidr_el1", + R::IdAa64dfr0El1 => "id_aa64dfr0_el1", + R::IdAa64dfr1El1 => "id_aa64dfr1_el1", + R::IdAa64isar0El1 => "id_aa64isar0_el1", + R::IdAa64isar1El1 => "id_aa64isar1_el1", + R::IdAa64isar2El1 => "id_aa64isar2_el1", + R::IdAa64mmfr0El1 => "id_aa64mmfr0_el1", + R::IdAa64mmfr1El1 => "id_aa64mmfr1_el1", + R::IdAa64mmfr2El1 => "id_aa64mmfr2_el1", + R::IdAa64mmfr3El1 => "id_aa64mmfr3_el1", + R::IdAa64mmfr4El1 => "id_aa64mmfr4_el1", + R::Nzcv => "nzcv", + R::DaifSet => "daifset", + R::DaifClear => "daifclr", + R::SpSelSet => "spsel", + R::PanSet => "pan", + R::Apple(slot) => { + let (op1, crm, op2) = APPLE[usize::from(slot) % APPLE.len()]; + return generic_register(f, 3, op1, 15, crm, op2); + } + R::AppleBank(slot) => { + let op1 = ((slot >> 7) & 7) as u8; + let crm = ((slot >> 3) & 15) as u8; + let op2 = (slot & 7) as u8; + return generic_register(f, 3, op1, 15, crm, op2); + } + R::AppleZero { op1, crm, op2 } => return generic_register(f, 3, op1, 15, crm, op2), + R::RazWi { crm, op2 } => { + let name = ["dbgbvr", "dbgbcr", "dbgwvr", "dbgwcr"][usize::from(op2 & 3)]; + return write!(f, "{name}{crm}_el1"); + } + R::PacKey(slot) => { + let (key, half) = [ + ("apia", "lo"), + ("apia", "hi"), + ("apib", "lo"), + ("apib", "hi"), + ("apda", "lo"), + ("apda", "hi"), + ("apdb", "lo"), + ("apdb", "hi"), + ("apga", "lo"), + ("apga", "hi"), + ][usize::from(slot) % 10]; + return write!(f, "{key}key{half}_el1"); + } + }; + f.write_str(name) +} + +fn system_register_move(f: &mut Formatter<'_>, system: &decode::System) -> fmt::Result { + use SystemRegister as R; + if system.read { + f.write_str("mrs ")?; + gpr(f, Width::X64, system.rt)?; + f.write_str(", ")?; + return register_name(f, system.register); + } + f.write_str("msr ")?; + register_name(f, system.register)?; + f.write_str(", ")?; + match system.register { + R::DaifSet | R::DaifClear | R::SpSelSet | R::PanSet => { + write!(f, "#{:#x}", system.immediate) + } + _ => gpr(f, Width::X64, system.rt), + } +} + +/// The key letter of `braa`/`retab`/`eretaa`: `a` then `a` or `b`. +fn branch_key(key: KeyId) -> &'static str { + match key { + KeyId::Ib => "ab", + _ => "aa", + } +} + +/// The key as `` (`ga` for the generic key); with a zero modifier the +/// `z` goes between the two letters (`paciza`, `autdzb`). +fn key_name(f: &mut Formatter<'_>, key: KeyId, zero: bool) -> fmt::Result { + let (kind, which) = match key { + KeyId::Ia => ("i", "a"), + KeyId::Ib => ("i", "b"), + KeyId::Da => ("d", "a"), + KeyId::Db => ("d", "b"), + KeyId::Ga => ("g", "a"), + }; + write!(f, "{kind}{}{which}", if zero { "z" } else { "" }) +} + +/// `sp` for register 31, else `xN`: the modifier register of pointer authentication. +fn modifier_register(f: &mut Formatter<'_>, r: u8) -> fmt::Result { + if r == 31 { + f.write_str("sp") + } else { + write!(f, "x{r}") + } +} + +fn pauth_op(f: &mut Formatter<'_>, op: pauth::Op) -> fmt::Result { + use pauth::Op::{Auth, Branch, ExceptionReturn, Pacga, Return, Sign, Strip}; + match op { + Sign { + key, + reg, + modifier, + hint, + } + | Auth { + key, + reg, + modifier, + hint, + } => { + f.write_str(if matches!(op, Sign { .. }) { + "pac" + } else { + "aut" + })?; + if hint { + // `paciaz`, `paciasp`, `pacia1716`: no operands. + key_name(f, key, false)?; + return f.write_str(match modifier { + Modifier::Zero => "z", + Modifier::Reg(31) => "sp", + Modifier::Reg(_) => "1716", + }); + } + key_name(f, key, matches!(modifier, Modifier::Zero))?; + f.write_str(" ")?; + gpr(f, Width::X64, reg)?; + if let Modifier::Reg(m) = modifier { + f.write_str(", ")?; + modifier_register(f, m)?; + } + Ok(()) + } + Strip { hint: true, .. } => f.write_str("xpaclri"), + Strip { data, reg, .. } => { + f.write_str(if data { "xpacd " } else { "xpaci " })?; + gpr(f, Width::X64, reg) + } + Pacga { rd, rn, modifier } => { + f.write_str("pacga ")?; + gpr(f, Width::X64, rd)?; + f.write_str(", ")?; + gpr(f, Width::X64, rn)?; + f.write_str(", ")?; + match modifier { + Modifier::Reg(m) => modifier_register(f, m), + Modifier::Zero => f.write_str("xzr"), + } + } + Branch { + key, + link, + target, + modifier, + } => { + write!(f, "{}{}", if link { "blr" } else { "br" }, branch_key(key))?; + if matches!(modifier, Modifier::Zero) { + f.write_str("z")?; + } + f.write_str(" ")?; + gpr(f, Width::X64, target)?; + if let Modifier::Reg(m) = modifier { + f.write_str(", ")?; + modifier_register(f, m)?; + } + Ok(()) + } + Return { key } => write!(f, "ret{}", branch_key(key)), + ExceptionReturn { key } => write!(f, "eret{}", branch_key(key)), + } +} + +#[cfg(test)] +mod tests { + use crate::format::{word, word_at}; + + fn check(w: u32, expected: &str) { + assert_eq!( + std::string::ToString::to_string(&word(w)), + expected, + "{w:#010x}" + ); + } + + fn check_at(w: u32, pc: u64, expected: &str) { + assert_eq!( + std::string::ToString::to_string(&word_at(w, pc)), + expected, + "{w:#010x} at {pc:#x}" + ); + } + + #[test] + fn relative_branches_without_an_address() { + check(0x14000002, "b .+8"); + check(0x17ffffff, "b .-4"); + check(0x94000040, "bl .+256"); + check(0x97fffffe, "bl .-8"); + check(0x54000040, "b.eq .+8"); + check(0x54ffffe1, "b.ne .-4"); + check(0x34000040, "cbz w0, .+8"); + check(0xb5fffff5, "cbnz x21, .-4"); + check(0x36180040, "tbz w0, #3, .+8"); + check(0xb7f80041, "tbnz x1, #63, .+8"); + } + + #[test] + fn relative_branches_at_an_address() { + check_at(0x14000002, 0x10100000, "b 0x10100008"); + check_at(0x94000040, 0x10100004, "bl 0x10100104"); + check_at(0x17ffffff, 0x10100008, "b 0x10100004"); + check_at(0x97fffffe, 0x1010000c, "bl 0x10100004"); + check_at(0x15ffffff, 0x10100010, "b 0x1810000c"); + check_at(0x96000000, 0x10100014, "bl 0x8100014"); + check_at(0x54000040, 0x10100018, "b.eq 0x10100020"); + check_at(0x54ffffe1, 0x1010001c, "b.ne 0x10100018"); + check_at(0x543fffe2, 0x10100020, "b.cs 0x1018001c"); + check_at(0x54800003, 0x10100024, "b.cc 0x10000024"); + check_at(0x54000024, 0x10100028, "b.mi 0x1010002c"); + check_at(0x54000025, 0x1010002c, "b.pl 0x10100030"); + check_at(0x54000026, 0x10100030, "b.vs 0x10100034"); + check_at(0x54000027, 0x10100034, "b.vc 0x10100038"); + check_at(0x54000028, 0x10100038, "b.hi 0x1010003c"); + check_at(0x54000029, 0x1010003c, "b.ls 0x10100040"); + check_at(0x5400002a, 0x10100040, "b.ge 0x10100044"); + check_at(0x5400002b, 0x10100044, "b.lt 0x10100048"); + check_at(0x5400002c, 0x10100048, "b.gt 0x1010004c"); + check_at(0x5400002d, 0x1010004c, "b.le 0x10100050"); + check_at(0x34000040, 0x10100050, "cbz w0, 0x10100058"); + check_at(0x35ffffc5, 0x10100054, "cbnz w5, 0x1010004c"); + check_at(0xb400009f, 0x10100058, "cbz xzr, 0x10100068"); + check_at(0xb580001e, 0x1010005c, "cbnz x30, 0x1000005c"); + check_at(0x357fffff, 0x10100060, "cbnz wzr, 0x1020005c"); + check_at(0x36000040, 0x10100064, "tbz w0, #0, 0x1010006c"); + check_at(0x372fffc3, 0x10100068, "tbnz w3, #5, 0x10100060"); + check_at(0x36f8ffff, 0x1010006c, "tbz wzr, #31, 0x10102068"); + check_at(0xb704001e, 0x10100070, "tbnz x30, #32, 0x100f8070"); + check_at(0xb6f80205, 0x10100074, "tbz x5, #63, 0x101000b4"); + check_at(0xb7080020, 0x10100078, "tbnz x0, #33, 0x1010007c"); + } + + #[test] + fn hints() { + check(0xd503201f, "nop"); + check(0xd503203f, "yield"); + check(0xd503205f, "wfe"); + check(0xd503207f, "wfi"); + check(0xd503209f, "sev"); + check(0xd50320bf, "sevl"); + check(0xd50320df, "dgh"); + check(0xd50320ff, "xpaclri"); + check(0xd503211f, "pacia1716"); + check(0xd503215f, "pacib1716"); + check(0xd503219f, "autia1716"); + check(0xd50321df, "autib1716"); + check(0xd50321ff, "hint #0xf"); + check(0xd503221f, "esb"); + check(0xd503223f, "psb csync"); + check(0xd503225f, "tsb csync"); + check(0xd503227f, "gcsb dsync"); + check(0xd503229f, "csdb"); + check(0xd50322bf, "hint #0x15"); + check(0xd50322df, "clrbhb"); + check(0xd50322ff, "hint #0x17"); + check(0xd503231f, "paciaz"); + check(0xd503233f, "paciasp"); + check(0xd503235f, "pacibz"); + check(0xd503237f, "pacibsp"); + check(0xd503239f, "autiaz"); + check(0xd50323bf, "autiasp"); + check(0xd50323df, "autibz"); + check(0xd50323ff, "autibsp"); + check(0xd503241f, "bti r"); + check(0xd503243f, "hint #0x21"); + check(0xd503245f, "bti c"); + check(0xd503249f, "bti j"); + check(0xd50324df, "bti jc"); + check(0xd503251f, "chkfeat x16"); + check(0xd503261f, "stshh keep"); + check(0xd503263f, "stshh strm"); + check(0xd503265f, "shuh"); + check(0xd503267f, "shuh ph"); + check(0xd503269f, "stcph"); + check(0xd50326bf, "hint #0x35"); + check(0xd5032fff, "hint #0x7f"); + } + + #[test] + fn barriers_and_clrex() { + check(0xd503305f, "clrex #0x0"); + check(0xd503315f, "clrex #0x1"); + check(0xd503325f, "clrex #0x2"); + check(0xd503335f, "clrex #0x3"); + check(0xd503345f, "clrex #0x4"); + check(0xd503355f, "clrex #0x5"); + check(0xd503365f, "clrex #0x6"); + check(0xd503375f, "clrex #0x7"); + check(0xd503385f, "clrex #0x8"); + check(0xd503395f, "clrex #0x9"); + check(0xd5033a5f, "clrex #0xa"); + check(0xd5033b5f, "clrex #0xb"); + check(0xd5033c5f, "clrex #0xc"); + check(0xd5033d5f, "clrex #0xd"); + check(0xd5033e5f, "clrex #0xe"); + check(0xd5033f5f, "clrex"); + check(0xd503309f, "ssbb"); + check(0xd503319f, "dsb oshld"); + check(0xd503329f, "dsb oshst"); + check(0xd503339f, "dsb osh"); + check(0xd503349f, "pssbb"); + check(0xd503359f, "dsb nshld"); + check(0xd503369f, "dsb nshst"); + check(0xd503379f, "dsb nsh"); + check(0xd503389f, "dsb #0x08"); + check(0xd503399f, "dsb ishld"); + check(0xd5033a9f, "dsb ishst"); + check(0xd5033b9f, "dsb ish"); + check(0xd5033c9f, "dsb #0x0c"); + check(0xd5033d9f, "dsb ld"); + check(0xd5033e9f, "dsb st"); + check(0xd5033f9f, "dsb sy"); + check(0xd50330bf, "dmb #0x00"); + check(0xd50331bf, "dmb oshld"); + check(0xd50332bf, "dmb oshst"); + check(0xd50333bf, "dmb osh"); + check(0xd50334bf, "dmb #0x04"); + check(0xd50335bf, "dmb nshld"); + check(0xd50336bf, "dmb nshst"); + check(0xd50337bf, "dmb nsh"); + check(0xd50338bf, "dmb #0x08"); + check(0xd50339bf, "dmb ishld"); + check(0xd5033abf, "dmb ishst"); + check(0xd5033bbf, "dmb ish"); + check(0xd5033cbf, "dmb #0x0c"); + check(0xd5033dbf, "dmb ld"); + check(0xd5033ebf, "dmb st"); + check(0xd5033fbf, "dmb sy"); + check(0xd50330df, "isb #0x0"); + check(0xd50331df, "isb #0x1"); + check(0xd50332df, "isb #0x2"); + check(0xd50333df, "isb #0x3"); + check(0xd50334df, "isb #0x4"); + check(0xd50335df, "isb #0x5"); + check(0xd50336df, "isb #0x6"); + check(0xd50337df, "isb #0x7"); + check(0xd50338df, "isb #0x8"); + check(0xd50339df, "isb #0x9"); + check(0xd5033adf, "isb #0xa"); + check(0xd5033bdf, "isb #0xb"); + check(0xd5033cdf, "isb #0xc"); + check(0xd5033ddf, "isb #0xd"); + check(0xd5033edf, "isb #0xe"); + check(0xd5033fdf, "isb"); + } + + #[test] + fn every_system_register_reads() { + check(0xd5384100, "mrs x0, sp_el0"); + check(0xd53c4100, "mrs x0, sp_el1"); + check(0xd53e4100, "mrs x0, sp_el2"); + check(0xd53c1000, "mrs x0, sctlr_el2"); + check(0xd53c2040, "mrs x0, tcr_el2"); + check(0xd53c2000, "mrs x0, ttbr0_el2"); + check(0xd53c2020, "mrs x0, ttbr1_el2"); + check(0xd53ca200, "mrs x0, mair_el2"); + check(0xd53cc000, "mrs x0, vbar_el2"); + check(0xd53c4020, "mrs x0, elr_el2"); + check(0xd53c4000, "mrs x0, spsr_el2"); + check(0xd53c5200, "mrs x0, esr_el2"); + check(0xd53c6000, "mrs x0, far_el2"); + check(0xd53c1100, "mrs x0, hcr_el2"); + check(0xd53cd040, "mrs x0, tpidr_el2"); + check(0xd53ce060, "mrs x0, cntvoff_el2"); + check(0xd5384240, "mrs x0, currentel"); + check(0xd5384000, "mrs x0, spsr_el1"); + check(0xd5384020, "mrs x0, elr_el1"); + check(0xd5385200, "mrs x0, esr_el1"); + check(0xd5386000, "mrs x0, far_el1"); + check(0xd5387400, "mrs x0, par_el1"); + check(0xd538c000, "mrs x0, vbar_el1"); + check(0xd5381040, "mrs x0, cpacr_el1"); + check(0xd5381000, "mrs x0, sctlr_el1"); + check(0xd5382000, "mrs x0, ttbr0_el1"); + check(0xd5382020, "mrs x0, ttbr1_el1"); + check(0xd5382040, "mrs x0, tcr_el1"); + check(0xd538a200, "mrs x0, mair_el1"); + check(0xd53bd040, "mrs x0, tpidr_el0"); + check(0xd53bd060, "mrs x0, tpidrro_el0"); + check(0xd53be040, "mrs x0, cntvct_el0"); + check(0xd53be320, "mrs x0, cntv_ctl_el0"); + check(0xd53be340, "mrs x0, cntv_cval_el0"); + check(0xd53be300, "mrs x0, cntv_tval_el0"); + check(0xd538d080, "mrs x0, tpidr_el1"); + check(0xd538d020, "mrs x0, contextidr_el1"); + check(0xd5300240, "mrs x0, mdscr_el1"); + check(0xd5301080, "mrs x0, oslar_el1"); + check(0xd5301380, "mrs x0, osdlr_el1"); + check(0xd5301180, "mrs x0, oslsr_el1"); + check(0xd538e100, "mrs x0, cntkctl_el1"); + check(0xd53b4400, "mrs x0, fpcr"); + check(0xd53b4420, "mrs x0, fpsr"); + check(0xd53b4220, "mrs x0, daif"); + check(0xd5384200, "mrs x0, spsel"); + check(0xd5384260, "mrs x0, pan"); + check(0xd538c100, "mrs x0, isr_el1"); + check(0xd538cca0, "mrs x0, icc_sre_el1"); + check(0xd5384600, "mrs x0, icc_pmr_el1"); + check(0xd538c860, "mrs x0, icc_bpr0_el1"); + check(0xd538cc80, "mrs x0, icc_ctlr_el1"); + check(0xd538ccc0, "mrs x0, icc_igrpen0_el1"); + check(0xd538cce0, "mrs x0, icc_igrpen1_el1"); + check(0xd538cc60, "mrs x0, icc_bpr1_el1"); + check(0xd53be220, "mrs x0, cntp_ctl_el0"); + check(0xd53be020, "mrs x0, cntpct_el0"); + check(0xd53be200, "mrs x0, cntp_tval_el0"); + check(0xd53be240, "mrs x0, cntp_cval_el0"); + check(0xd538cc00, "mrs x0, icc_iar1_el1"); + check(0xd538cc20, "mrs x0, icc_eoir1_el1"); + check(0xd5381020, "mrs x0, actlr_el1"); + check(0xd5380000, "mrs x0, midr_el1"); + check(0xd53800a0, "mrs x0, mpidr_el1"); + check(0xd53800c0, "mrs x0, revidr_el1"); + check(0xd53b0020, "mrs x0, ctr_el0"); + check(0xd53b00e0, "mrs x0, dczid_el0"); + check(0xd53be000, "mrs x0, cntfrq_el0"); + check(0xd5390020, "mrs x0, clidr_el1"); + check(0xd5390000, "mrs x0, ccsidr_el1"); + check(0xd53a0000, "mrs x0, csselr_el1"); + check(0xd5380400, "mrs x0, id_aa64pfr0_el1"); + check(0xd5380420, "mrs x0, id_aa64pfr1_el1"); + check(0xd5380440, "mrs x0, id_aa64pfr2_el1"); + check(0xd5380480, "mrs x0, id_aa64zfr0_el1"); + check(0xd53804a0, "mrs x0, id_aa64smfr0_el1"); + check(0xd53804e0, "mrs x0, id_aa64fpfr0_el1"); + check(0xd5380660, "mrs x0, id_aa64isar3_el1"); + check(0xd53900e0, "mrs x0, aidr_el1"); + check(0xd5380500, "mrs x0, id_aa64dfr0_el1"); + check(0xd5380520, "mrs x0, id_aa64dfr1_el1"); + check(0xd5380600, "mrs x0, id_aa64isar0_el1"); + check(0xd5380620, "mrs x0, id_aa64isar1_el1"); + check(0xd5380640, "mrs x0, id_aa64isar2_el1"); + check(0xd5380700, "mrs x0, id_aa64mmfr0_el1"); + check(0xd5380720, "mrs x0, id_aa64mmfr1_el1"); + check(0xd5380740, "mrs x0, id_aa64mmfr2_el1"); + check(0xd5380760, "mrs x0, id_aa64mmfr3_el1"); + check(0xd5380780, "mrs x0, id_aa64mmfr4_el1"); + check(0xd53b4200, "mrs x0, nzcv"); + } + + #[test] + fn every_system_register_writes() { + check(0xd5184100, "msr sp_el0, x0"); + check(0xd51c4100, "msr sp_el1, x0"); + check(0xd51e4100, "msr sp_el2, x0"); + check(0xd51c1000, "msr sctlr_el2, x0"); + check(0xd51c2040, "msr tcr_el2, x0"); + check(0xd51c2000, "msr ttbr0_el2, x0"); + check(0xd51c2020, "msr ttbr1_el2, x0"); + check(0xd51ca200, "msr mair_el2, x0"); + check(0xd51cc000, "msr vbar_el2, x0"); + check(0xd51c4020, "msr elr_el2, x0"); + check(0xd51c4000, "msr spsr_el2, x0"); + check(0xd51c5200, "msr esr_el2, x0"); + check(0xd51c6000, "msr far_el2, x0"); + check(0xd51c1100, "msr hcr_el2, x0"); + check(0xd51cd040, "msr tpidr_el2, x0"); + check(0xd51ce060, "msr cntvoff_el2, x0"); + check(0xd5184000, "msr spsr_el1, x0"); + check(0xd5184020, "msr elr_el1, x0"); + check(0xd5185200, "msr esr_el1, x0"); + check(0xd5186000, "msr far_el1, x0"); + check(0xd5187400, "msr par_el1, x0"); + check(0xd518c000, "msr vbar_el1, x0"); + check(0xd5181040, "msr cpacr_el1, x0"); + check(0xd5181000, "msr sctlr_el1, x0"); + check(0xd5182000, "msr ttbr0_el1, x0"); + check(0xd5182020, "msr ttbr1_el1, x0"); + check(0xd5182040, "msr tcr_el1, x0"); + check(0xd518a200, "msr mair_el1, x0"); + check(0xd51bd040, "msr tpidr_el0, x0"); + check(0xd51bd060, "msr tpidrro_el0, x0"); + check(0xd51be040, "msr cntvct_el0, x0"); + check(0xd51be320, "msr cntv_ctl_el0, x0"); + check(0xd51be340, "msr cntv_cval_el0, x0"); + check(0xd51be300, "msr cntv_tval_el0, x0"); + check(0xd518d080, "msr tpidr_el1, x0"); + check(0xd518d020, "msr contextidr_el1, x0"); + check(0xd5100240, "msr mdscr_el1, x0"); + check(0xd5101080, "msr oslar_el1, x0"); + check(0xd5101380, "msr osdlr_el1, x0"); + check(0xd5101180, "msr oslsr_el1, x0"); + check(0xd518e100, "msr cntkctl_el1, x0"); + check(0xd51b4400, "msr fpcr, x0"); + check(0xd51b4420, "msr fpsr, x0"); + check(0xd51b4220, "msr daif, x0"); + check(0xd5184200, "msr spsel, x0"); + check(0xd5184260, "msr pan, x0"); + check(0xd518cca0, "msr icc_sre_el1, x0"); + check(0xd5184600, "msr icc_pmr_el1, x0"); + check(0xd518c860, "msr icc_bpr0_el1, x0"); + check(0xd518cc80, "msr icc_ctlr_el1, x0"); + check(0xd518ccc0, "msr icc_igrpen0_el1, x0"); + check(0xd518cce0, "msr icc_igrpen1_el1, x0"); + check(0xd518cc60, "msr icc_bpr1_el1, x0"); + check(0xd51be220, "msr cntp_ctl_el0, x0"); + check(0xd51be020, "msr cntpct_el0, x0"); + check(0xd51be200, "msr cntp_tval_el0, x0"); + check(0xd51be240, "msr cntp_cval_el0, x0"); + check(0xd518cc00, "msr icc_iar1_el1, x0"); + check(0xd518cc20, "msr icc_eoir1_el1, x0"); + check(0xd5181020, "msr actlr_el1, x0"); + check(0xd51a0000, "msr csselr_el1, x0"); + check(0xd51b4200, "msr nzcv, x0"); + } + + #[test] + fn system_register_edge_cases() { + check(0xd5034fdf, "msr daifset, #0xf"); + check(0xd50342df, "msr daifset, #0x2"); + check(0xd5034cdf, "msr daifset, #0xc"); + check(0xd5034fff, "msr daifclr, #0xf"); + check(0xd50342ff, "msr daifclr, #0x2"); + check(0xd500409f, "msr pan, #0x0"); + check(0xd500419f, "msr pan, #0x1"); + check(0xd50040bf, "msr spsel, #0x0"); + check(0xd50041bf, "msr spsel, #0x1"); + check(0xd5382100, "mrs x0, apiakeylo_el1"); + check(0xd5382120, "mrs x0, apiakeyhi_el1"); + check(0xd5382140, "mrs x0, apibkeylo_el1"); + check(0xd5382160, "mrs x0, apibkeyhi_el1"); + check(0xd5382200, "mrs x0, apdakeylo_el1"); + check(0xd5382300, "mrs x0, apgakeylo_el1"); + check(0xd5382320, "mrs x0, apgakeyhi_el1"); + } + + #[test] + fn cache_maintenance() { + check(0xd5087100, "ic ialluis"); + check(0xd5087500, "ic iallu"); + check(0xd5087620, "dc ivac, x0"); + check(0xd5087640, "dc isw, x0"); + check(0xd5087660, "dc igvac, x0"); + check(0xd5087680, "dc igsw, x0"); + check(0xd50876a0, "dc igdvac, x0"); + check(0xd50876c0, "dc igdsw, x0"); + check(0xd5087a40, "dc csw, x0"); + check(0xd5087a80, "dc cgsw, x0"); + check(0xd5087ac0, "dc cgdsw, x0"); + check(0xd5087e40, "dc cisw, x0"); + check(0xd5087e80, "dc cigsw, x0"); + check(0xd5087ec0, "dc cigdsw, x0"); + check(0xd50b7520, "ic ivau, x0"); + check(0xd50b7a20, "dc cvac, x0"); + check(0xd50b7a60, "dc cgvac, x0"); + check(0xd50b7aa0, "dc cgdvac, x0"); + check(0xd50b7b00, "dc cvaoc, x0"); + check(0xd50b7b20, "dc cvau, x0"); + check(0xd50b7be0, "dc cgdvaoc, x0"); + check(0xd50b7e20, "dc civac, x0"); + check(0xd50b7e60, "dc cigvac, x0"); + check(0xd50b7ea0, "dc cigdvac, x0"); + check(0xd50c7e00, "dc cipae, x0"); + check(0xd50c7ee0, "dc cigdpae, x0"); + check(0xd50e7e20, "dc cipapa, x0"); + check(0xd50e7ea0, "dc cigdpapa, x0"); + check(0xd508711f, "ic ialluis"); + check(0xd508751f, "ic iallu"); + check(0xd508763f, "dc ivac, xzr"); + check(0xd508765f, "dc isw, xzr"); + check(0xd508767f, "dc igvac, xzr"); + check(0xd508769f, "dc igsw, xzr"); + check(0xd50876bf, "dc igdvac, xzr"); + check(0xd50876df, "dc igdsw, xzr"); + check(0xd5087a5f, "dc csw, xzr"); + check(0xd5087a9f, "dc cgsw, xzr"); + check(0xd5087adf, "dc cgdsw, xzr"); + check(0xd5087e5f, "dc cisw, xzr"); + check(0xd5087e9f, "dc cigsw, xzr"); + check(0xd5087edf, "dc cigdsw, xzr"); + check(0xd50b753f, "ic ivau, xzr"); + check(0xd50b7a3f, "dc cvac, xzr"); + check(0xd50b7a7f, "dc cgvac, xzr"); + check(0xd50b7abf, "dc cgdvac, xzr"); + check(0xd50b7b1f, "dc cvaoc, xzr"); + check(0xd50b7b3f, "dc cvau, xzr"); + check(0xd50b7bff, "dc cgdvaoc, xzr"); + check(0xd50b7e3f, "dc civac, xzr"); + check(0xd50b7e7f, "dc cigvac, xzr"); + check(0xd50b7ebf, "dc cigdvac, xzr"); + check(0xd50c7e1f, "dc cipae, xzr"); + check(0xd50c7eff, "dc cigdpae, xzr"); + check(0xd50e7e3f, "dc cipapa, xzr"); + check(0xd50e7ebf, "dc cigdpapa, xzr"); + } + + #[test] + fn tlbi_named() { + check(0xd5088100, "tlbi vmalle1os, x0"); + check(0xd50881e0, "tlbi vaale1os, x0"); + check(0xd5088300, "tlbi vmalle1is, x0"); + check(0xd50883e0, "tlbi vaale1is, x0"); + check(0xd5088620, "tlbi rvae1, x0"); + check(0xd5088720, "tlbi vae1, x0"); + check(0xd50c8020, "tlbi ipas2e1is, x0"); + check(0xd50c8120, "tlbi vae2os, x0"); + check(0xd50c8240, "tlbi vmallws2e1is, x0"); + check(0xd50c83a0, "tlbi vale2is, x0"); + check(0xd50c8460, "tlbi ripas2e1os, x0"); + check(0xd50c8520, "tlbi rvae2os, x0"); + check(0xd50c86a0, "tlbi rvale2, x0"); + check(0xd50c87c0, "tlbi vmalls12e1"); + check(0xd50e8220, "tlbi rvae3is, x0"); + check(0xd50e8460, "tlbi rpaos, x0"); + check(0xd50e86a0, "tlbi rvale3, x0"); + check(0xd508811f, "tlbi vmalle1os"); + check(0xd50881ff, "tlbi vaale1os, xzr"); + check(0xd508831f, "tlbi vmalle1is"); + check(0xd50883ff, "tlbi vaale1is, xzr"); + check(0xd508863f, "tlbi rvae1, xzr"); + check(0xd508873f, "tlbi vae1, xzr"); + check(0xd50c803f, "tlbi ipas2e1is, xzr"); + check(0xd50c813f, "tlbi vae2os, xzr"); + check(0xd50c825f, "tlbi vmallws2e1is"); + check(0xd50c83bf, "tlbi vale2is, xzr"); + check(0xd50c847f, "tlbi ripas2e1os, xzr"); + check(0xd50c853f, "tlbi rvae2os, xzr"); + check(0xd50c86bf, "tlbi rvale2, xzr"); + check(0xd50c87df, "tlbi vmalls12e1"); + check(0xd50e823f, "tlbi rvae3is, xzr"); + check(0xd50e847f, "tlbi rpaos, xzr"); + check(0xd50e86bf, "tlbi rvale3, xzr"); + } + + #[test] + fn generic_sys() { + check(0xd5087120, "sys #0, C7, C1, #1, x0"); + check(0xd508713f, "sys #0, C7, C1, #1"); + check(0xd50a883f, "sys #2, C8, C8, #1"); + check(0xd50a8840, "sys #2, C8, C8, #2, x0"); + check(0xd50f8fff, "sys #7, C8, C15, #7"); + check(0xd50f8fe0, "sys #7, C8, C15, #7, x0"); + check(0xd50b7420, "dc zva, x0"); + check(0xd50b743f, "dc zva, xzr"); + check(0xd5087800, "at s1e1r, x0"); + check(0xd5087820, "at s1e1w, x0"); + check(0xd5087840, "at s1e0r, x0"); + check(0xd508785f, "at s1e0r, xzr"); + check(0xd5087860, "at s1e0w, x0"); + } + + #[test] + fn exceptions() { + check(0xd4000002, "hvc #0x0"); + check(0xd4000001, "svc #0x0"); + check(0xd4001001, "svc #0x80"); + check(0xd4200020, "brk #0x1"); + check(0xd4210000, "brk #0x800"); + check(0xd43fffe0, "brk #0xffff"); + check(0xd69f03e0, "eret"); + } + + #[test] + fn pointer_authentication() { + check(0xdac10000, "pacia x0, x0"); + check(0xdac10020, "pacia x0, x1"); + check(0xdac103e0, "pacia x0, sp"); + check(0xdac10400, "pacib x0, x0"); + check(0xdac10800, "pacda x0, x0"); + check(0xdac10c00, "pacdb x0, x0"); + check(0xdac11000, "autia x0, x0"); + check(0xdac11400, "autib x0, x0"); + check(0xdac11800, "autda x0, x0"); + check(0xdac11c00, "autdb x0, x0"); + check(0xdac113e0, "autia x0, sp"); + check(0xdac123e0, "paciza x0"); + check(0xdac127e0, "pacizb x0"); + check(0xdac12be0, "pacdza x0"); + check(0xdac12fe0, "pacdzb x0"); + check(0xdac133e0, "autiza x0"); + check(0xdac137e0, "autizb x0"); + check(0xdac13be0, "autdza x0"); + check(0xdac13fe0, "autdzb x0"); + check(0xdac143e0, "xpaci x0"); + check(0x9ac033e0, "pacga x0, xzr, x0"); + check(0x9adf3000, "pacga x0, x0, sp"); + check(0xd71f0800, "braa x0, x0"); + check(0xd71f0fe3, "brab xzr, x3"); + check(0xd65f0bff, "retaa"); + check(0xd65f0fff, "retab"); + check(0xd69f0bff, "eretaa"); + check(0xd69f0fff, "eretab"); + check(0xd503231f, "paciaz"); + check(0xd503233f, "paciasp"); + check(0xd503235f, "pacibz"); + check(0xd503237f, "pacibsp"); + check(0xd503239f, "autiaz"); + check(0xd50323bf, "autiasp"); + check(0xd50323df, "autibz"); + check(0xd50323ff, "autibsp"); + check(0xd503211f, "pacia1716"); + check(0xd503215f, "pacib1716"); + check(0xd503219f, "autia1716"); + check(0xd50321df, "autib1716"); + check(0xd50320ff, "xpaclri"); + // The register forms of the same operations are not the hint spellings. + check(0xdac103fe, "pacia x30, sp"); + check(0xdac10211, "pacia x17, x16"); + check(0xdac10611, "pacib x17, x16"); + check(0xdac113fe, "autia x30, sp"); + check(0xdac11211, "autia x17, x16"); + check(0xdac11611, "autib x17, x16"); + check(0xdac123fe, "paciza x30"); + check(0xdac127fe, "pacizb x30"); + check(0xdac133fe, "autiza x30"); + check(0xdac137fe, "autizb x30"); + check(0xdac143fe, "xpaci x30"); + check(0xdac147e0, "xpacd x0"); + check(0xdac147fe, "xpacd x30"); + } + + #[test] + fn register_branches() { + check(0xd61f0000, "br x0"); + check(0xd61f0220, "br x17"); + check(0xd61f03e0, "br xzr"); + check(0xd63f0000, "blr x0"); + check(0xd63f03c0, "blr x30"); + check(0xd63f03e0, "blr xzr"); + check(0xd65f0000, "ret x0"); + check(0xd65f03c0, "ret"); + check(0xd65f0220, "ret x17"); + check(0xd65f03e0, "ret xzr"); + } + + #[test] + fn debug_and_apple_zero_registers_keep_their_spelling() { + check(0xd5300080, "mrs x0, dbgbvr0_el1"); + check(0xd53000a0, "mrs x0, dbgbcr0_el1"); + check(0xd53000c0, "mrs x0, dbgwvr0_el1"); + check(0xd53000e0, "mrs x0, dbgwcr0_el1"); + check(0xd5300fe0, "mrs x0, dbgwcr15_el1"); + check(0xd5300ac1, "mrs x1, dbgwvr10_el1"); + check(0xd5100180, "msr dbgbvr1_el1, x0"); + check(0xd51001a0, "msr dbgbcr1_el1, x0"); + check(0xd51001c0, "msr dbgwvr1_el1, x0"); + check(0xd51001e0, "msr dbgwcr1_el1, x0"); + check(0xd53df600, "mrs x0, s3_5_c15_c6_0"); + check(0xd53ff680, "mrs x0, s3_7_c15_c6_4"); + check(0xd539fd00, "mrs x0, s3_1_c15_c13_0"); + check(0xd53df120, "mrs x0, s3_5_c15_c1_1"); + } + + #[test] + fn implementation_defined_registers() { + check(0xd539f000, "mrs x0, s3_1_c15_c0_0"); + check(0xd539f600, "mrs x0, s3_1_c15_c6_0"); + check(0xd53afa00, "mrs x0, s3_2_c15_c10_0"); + check(0xd53df320, "mrs x0, s3_5_c15_c3_1"); + check(0xd53ff080, "mrs x0, s3_7_c15_c0_4"); + check(0xd539fd00, "mrs x0, s3_1_c15_c13_0"); + check(0xd53df120, "mrs x0, s3_5_c15_c1_1"); + check(0xd53df300, "mrs x0, s3_5_c15_c3_0"); + check(0xd53df700, "mrs x0, s3_5_c15_c7_0"); + check(0xd53cf3a0, "mrs x0, s3_4_c15_c3_5"); + check(0xd53effe0, "mrs x0, s3_6_c15_c15_7"); + check(0xd538f000, "mrs x0, s3_0_c15_c0_0"); + check(0xd538ffa0, "mrs x0, s3_0_c15_c15_5"); + check(0xd53bfa00, "mrs x0, s3_3_c15_c10_0"); + check(0xd539f840, "mrs x0, s3_1_c15_c8_2"); + check(0xd5382100, "mrs x0, apiakeylo_el1"); + check(0xd5382120, "mrs x0, apiakeyhi_el1"); + check(0xd5382140, "mrs x0, apibkeylo_el1"); + check(0xd5382160, "mrs x0, apibkeyhi_el1"); + check(0xd5382200, "mrs x0, apdakeylo_el1"); + check(0xd5382220, "mrs x0, apdakeyhi_el1"); + check(0xd5382240, "mrs x0, apdbkeylo_el1"); + check(0xd5382260, "mrs x0, apdbkeyhi_el1"); + check(0xd5382300, "mrs x0, apgakeylo_el1"); + check(0xd5382320, "mrs x0, apgakeyhi_el1"); + check(0xd519f000, "msr s3_1_c15_c0_0, x0"); + check(0xd519f600, "msr s3_1_c15_c6_0, x0"); + check(0xd51afa00, "msr s3_2_c15_c10_0, x0"); + check(0xd5182100, "msr apiakeylo_el1, x0"); + check(0xd5182120, "msr apiakeyhi_el1, x0"); + } +} diff --git a/src/isa/aarch64/src/format/vector.rs b/src/isa/aarch64/src/format/vector.rs new file mode 100644 index 0000000..50c4771 --- /dev/null +++ b/src/isa/aarch64/src/format/vector.rs @@ -0,0 +1,1033 @@ +//! Advanced SIMD data processing: modified immediates, moves, `dup`/`ins`/`umov`, +//! `ext`, `tbl`/`tbx`, the integer three-same group, compares against zero, and the +//! integer instruction forms of `crate::simd`. +//! +//! Owns `SimdImm`, `SimdCompareZero`, `SimdDup`, `SimdDupElement`, +//! `SimdInsert`, `SimdInsertElement`, `SimdExtract`, `SimdExt`, `SimdAlu`, `SimdTable` +//! and `Simd`. + +use super::Context; +use super::reg::{fp_scalar, gpr, vector}; +use crate::decode::{ + ImmCombine, ImmForm, Instruction, SimdAlu, SimdAluOp, SimdImmediate, SimdTable, Width, +}; +use crate::simd::{Long, Narrow, Op, Permute, Scalar, Zero}; +use core::fmt::{self, Formatter, Write}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + let _ = cx; + match *instruction { + Instruction::SimdImm(imm) => immediate(f, imm), + Instruction::SimdCompareZero(c) => { + f.write_str("cmeq ")?; + two_same(f, c.rd, c.rn, c.size, c.q)?; + f.write_str(", #0") + } + Instruction::SimdDup { rd, rn, esize, q } => { + f.write_str("dup ")?; + vector(f, rd, esize & 3, q)?; + f.write_str(", ")?; + gpr(f, lane_width(esize), rn) + } + Instruction::SimdDupElement(d) => { + f.write_str("dup ")?; + vector(f, d.rd, d.esize & 3, d.q)?; + f.write_str(", ")?; + element(f, d.rn, d.esize, d.index) + } + Instruction::SimdInsert(i) => { + f.write_str("mov ")?; + element(f, i.rd, i.esize, i.index)?; + f.write_str(", ")?; + gpr(f, lane_width(i.esize), i.rn) + } + Instruction::SimdInsertElement(i) => { + f.write_str("mov ")?; + element(f, i.rd, i.esize, i.dst)?; + f.write_str(", ")?; + element(f, i.rn, i.esize, i.src) + } + // `umov` of a word or doubleword lane is spelled `mov`. + Instruction::SimdExtract(e) => { + f.write_str(if e.esize & 3 >= 2 { "mov " } else { "umov " })?; + gpr(f, lane_width(e.esize), e.rd)?; + f.write_str(", ")?; + element(f, e.rn, e.esize, e.index) + } + Instruction::SimdExt(e) => { + f.write_str("ext ")?; + vector(f, e.rd, 0, e.q)?; + f.write_str(", ")?; + vector(f, e.rn, 0, e.q)?; + f.write_str(", ")?; + vector(f, e.rm, 0, e.q)?; + write!(f, ", #{}", e.imm) + } + Instruction::SimdAlu(alu) => simd_alu(f, alu), + Instruction::SimdTable(t) => table(f, t), + Instruction::Simd(op) => simd(f, op), + _ => unreachable!("{instruction:?} is not an Advanced SIMD vector instruction"), + } +} + +/// `X` for a doubleword lane, `W` for every narrower one. +fn lane_width(esize: u8) -> Width { + if esize & 3 == 3 { + Width::X64 + } else { + Width::W32 + } +} + +/// The element letter of `esize` (log2 of the byte width). +fn lane_letter(esize: u8) -> char { + ['b', 'h', 's', 'd'][usize::from(esize & 3)] +} + +/// One lane of a vector register: `v1.s[2]`. +fn element(f: &mut Formatter<'_>, r: u8, esize: u8, index: u8) -> fmt::Result { + write!(f, "v{r}.{}[{index}]", lane_letter(esize)) +} + +/// `vd.T, vn.T` with one arrangement. +fn two_same(f: &mut Formatter<'_>, rd: u8, rn: u8, size: u8, q: bool) -> fmt::Result { + vector(f, rd, size & 3, q)?; + f.write_str(", ")?; + vector(f, rn, size & 3, q) +} + +/// `vd.T, vn.T, vm.T` with one arrangement. +fn three_same(f: &mut Formatter<'_>, rd: u8, rn: u8, rm: u8, size: u8, q: bool) -> fmt::Result { + two_same(f, rd, rn, size, q)?; + f.write_str(", ")?; + vector(f, rm, size & 3, q) +} + +fn simd_alu(f: &mut Formatter<'_>, alu: SimdAlu) -> fmt::Result { + use SimdAluOp::*; + let SimdAlu { + op, + rd, + rn, + rm, + size, + q, + } = alu; + let size = size & 3; + let name = match op { + Add => "add", + Sub => "sub", + Mul => "mul", + Mla => "mla", + Mls => "mls", + And => "and", + Bic => "bic", + Orr if rn == rm => { + f.write_str("mov ")?; + return two_same(f, rd, rn, 0, q); + } + Orr => "orr", + Orn => "orn", + Eor => "eor", + Bsl => "bsl", + Bit => "bit", + Bif => "bif", + CmEq => "cmeq", + CmGt => "cmgt", + CmGe => "cmge", + CmHi => "cmhi", + CmHs => "cmhs", + CmTst => "cmtst", + UMin => "umin", + UMax => "umax", + SMin => "smin", + SMax => "smax", + UAbd => "uabd", + SAbd => "sabd", + SQAdd => "sqadd", + UQAdd => "uqadd", + SQSub => "sqsub", + UQSub => "uqsub", + SHAdd => "shadd", + UHAdd => "uhadd", + SHSub => "shsub", + UHSub => "uhsub", + SRHAdd => "srhadd", + URHAdd => "urhadd", + SSHl => "sshl", + USHl => "ushl", + SQShl => "sqshl", + UQShl => "uqshl", + SQRShl => "sqrshl", + UQRShl => "uqrshl", + AddV => "addv", + UMinV => "uminv", + UMaxV => "umaxv", + UAddLV => "uaddlv", + UMaxP => "umaxp", + SMaxP => "smaxp", + UMinP => "uminp", + SMinP => "sminp", + AddP => "addp", + }; + write!(f, "{name} ")?; + match op { + // Across lanes: a scalar result, as wide as the lanes (twice as wide for + // `uaddlv`), from a vector. + AddV | UMinV | UMaxV | UAddLV => { + let bytes = 1u8 << (size + u8::from(op == UAddLV)); + fp_scalar(f, bytes, rd)?; + f.write_str(", ")?; + vector(f, rn, size, q) + } + And | Bic | Orr | Orn | Eor | Bsl | Bit | Bif => three_same(f, rd, rn, rm, 0, q), + _ => three_same(f, rd, rn, rm, size, q), + } +} + +fn table(f: &mut Formatter<'_>, t: SimdTable) -> fmt::Result { + f.write_str(if t.extend { "tbx " } else { "tbl " })?; + vector(f, t.rd, 0, t.q)?; + f.write_str(", {")?; + vector(f, t.rn, 0, true)?; + if t.len & 3 != 0 { + f.write_str("-")?; + vector(f, t.rn.wrapping_add(t.len & 3) & 31, 0, true)?; + } + f.write_str("}, ")?; + vector(f, t.rm, 0, t.q) +} + +/// The mnemonic suffix of the upper-half (`2`) form of a long or narrow operation. +fn two(upper: bool) -> &'static str { + if upper { "2" } else { "" } +} + +fn simd(f: &mut Formatter<'_>, op: Op) -> fmt::Result { + match op { + Op::Permute { + kind, + size, + q, + rd, + rn, + rm, + } => { + f.write_str(match kind { + Permute::Uzp1 => "uzp1 ", + Permute::Uzp2 => "uzp2 ", + Permute::Zip1 => "zip1 ", + Permute::Zip2 => "zip2 ", + Permute::Trn1 => "trn1 ", + Permute::Trn2 => "trn2 ", + })?; + three_same(f, rd, rn, rm, size as u8, q) + } + Op::Reverse { + group, + size, + q, + rd, + rn, + } => { + write!(f, "rev{group} ")?; + two_same(f, rd, rn, size as u8, q) + } + Op::Count { q, rd, rn } => { + f.write_str("cnt ")?; + two_same(f, rd, rn, 0, q) + } + Op::Not { q, rd, rn } => { + f.write_str("mvn ")?; + two_same(f, rd, rn, 0, q) + } + Op::ReverseBits { q, rd, rn } => { + f.write_str("rbit ")?; + two_same(f, rd, rn, 0, q) + } + Op::CompareZero { + kind, + size, + q, + rd, + rn, + } => { + f.write_str(match kind { + Zero::Gt => "cmgt ", + Zero::Ge => "cmge ", + Zero::Le => "cmle ", + Zero::Lt => "cmlt ", + })?; + two_same(f, rd, rn, size as u8, q)?; + f.write_str(", #0") + } + Op::Abs { + negate, + size, + q, + rd, + rn, + } => { + f.write_str(if negate { "neg " } else { "abs " })?; + two_same(f, rd, rn, size as u8, q) + } + Op::Narrow { + kind, + size, + upper, + rd, + rn, + } => { + let name = match kind { + Narrow::Plain => "xtn", + Narrow::SignedSat => "sqxtn", + Narrow::UnsignedSat => "uqxtn", + Narrow::SignedToUnsignedSat => "sqxtun", + }; + let size = size as u8 & 3; + write!(f, "{name}{} ", two(upper))?; + vector(f, rd, size, upper)?; + f.write_str(", ")?; + vector(f, rn, size.wrapping_add(1) & 3, true) + } + Op::PairLong { + signed, + accumulate, + size, + q, + rd, + rn, + } => { + let size = size as u8 & 3; + write!( + f, + "{}{} ", + if signed { 's' } else { 'u' }, + if accumulate { "adalp" } else { "addlp" } + )?; + vector(f, rd, size.wrapping_add(1) & 3, q)?; + f.write_str(", ")?; + vector(f, rn, size, q) + } + Op::Long { + kind, + signed, + size, + upper, + rd, + rn, + rm, + } => { + let name = match kind { + Long::Add => "addl", + Long::AddWide => "addw", + Long::Sub => "subl", + Long::SubWide => "subw", + Long::Abd => "abdl", + Long::Mul => "mull", + Long::Mla => "mlal", + Long::Mls => "mlsl", + }; + let size = size as u8 & 3; + write!(f, "{}{name}{} ", if signed { 's' } else { 'u' }, two(upper))?; + vector(f, rd, size.wrapping_add(1) & 3, true)?; + f.write_str(", ")?; + if matches!(kind, Long::AddWide | Long::SubWide) { + vector(f, rn, size.wrapping_add(1) & 3, true)?; + } else { + vector(f, rn, size, upper)?; + } + f.write_str(", ")?; + vector(f, rm, size, upper) + } + Op::ScalarAddp { rd, rn } => write!(f, "addp d{rd}, v{rn}.2d"), + Op::ScalarDup { + size, + index, + rd, + rn, + } => { + let size = size as u8 & 3; + f.write_str("mov ")?; + fp_scalar(f, 1 << size, rd)?; + f.write_str(", ")?; + element(f, rn, size, index as u8) + } + Op::ScalarThree { kind, rd, rn, rm } => { + let name = match kind { + Scalar::Add => "add", + Scalar::Sub => "sub", + Scalar::Gt => "cmgt", + Scalar::Hi => "cmhi", + Scalar::Ge => "cmge", + Scalar::Hs => "cmhs", + Scalar::Eq => "cmeq", + Scalar::Tst => "cmtst", + }; + write!(f, "{name} d{rd}, d{rn}, d{rm}") + } + Op::ShiftImm { + right, + signed, + size, + q, + shift, + rd, + rn, + } => { + f.write_str(match (right, signed) { + (false, _) => "shl ", + (true, true) => "sshr ", + (true, false) => "ushr ", + })?; + two_same(f, rd, rn, size as u8, q)?; + write!(f, ", #{shift}") + } + Op::ShiftLong { + signed, + size, + shift, + upper, + rd, + rn, + } => { + let size = size as u8 & 3; + let sign = if signed { 's' } else { 'u' }; + if shift == 0 { + write!(f, "{sign}xtl{} ", two(upper))?; + } else { + write!(f, "{sign}shll{} ", two(upper))?; + } + vector(f, rd, size.wrapping_add(1) & 3, true)?; + f.write_str(", ")?; + vector(f, rn, size, upper)?; + if shift != 0 { + write!(f, ", #{shift}")?; + } + Ok(()) + } + Op::ShiftNarrow { + size, + shift, + upper, + rd, + rn, + } => { + let size = size as u8 & 3; + write!(f, "shrn{} ", two(upper))?; + vector(f, rd, size, upper)?; + f.write_str(", ")?; + vector(f, rn, size.wrapping_add(1) & 3, true)?; + write!(f, ", #{shift}") + } + } +} + +/// A floating-point immediate the way binutils prints it: C's `%.18e`. +fn float_immediate(f: &mut Formatter<'_>, value: f64) -> fmt::Result { + // `fmov` immediates are always finite; other bit patterns only come from + // hand-built values and print as C's `%f` would. + if !value.is_finite() { + return f.write_str(if value.is_nan() { + "nan" + } else if value < 0.0 { + "-inf" + } else { + "inf" + }); + } + let mut text = Text { + bytes: [0; 64], + len: 0, + }; + write!(text, "{value:.18e}")?; + let text = core::str::from_utf8(&text.bytes[..text.len]).map_err(|_| fmt::Error)?; + let (mantissa, exponent) = text.split_once('e').ok_or(fmt::Error)?; + let exponent: i32 = exponent.parse().map_err(|_| fmt::Error)?; + let sign = if exponent < 0 { '-' } else { '+' }; + write!(f, "{mantissa}e{sign}{:02}", exponent.unsigned_abs()) +} + +/// A fixed buffer to format a float into; every value printed here fits. +struct Text { + bytes: [u8; 64], + len: usize, +} + +impl Write for Text { + fn write_str(&mut self, s: &str) -> fmt::Result { + let end = self.len.checked_add(s.len()).ok_or(fmt::Error)?; + self.bytes + .get_mut(self.len..end) + .ok_or(fmt::Error)? + .copy_from_slice(s.as_bytes()); + self.len = end; + Ok(()) + } +} + +/// `name v., #[, #]`. +fn lane_form( + f: &mut Formatter<'_>, + name: &str, + rd: u8, + lanes: &str, + imm8: u8, + kind: &str, + shift: u8, +) -> fmt::Result { + write!(f, "{name} v{rd}.{lanes}, #{imm8:#x}")?; + if shift != 0 { + write!(f, ", {kind} #{shift}")?; + } + Ok(()) +} + +/// A half-precision value (as `fmov`'s expanded immediates are: always normal) as +/// a double. +fn half_to_f64(bits: u16) -> f64 { + let sign = if bits & 0x8000 != 0 { -1.0 } else { 1.0 }; + let exponent = u64::from((bits >> 10) & 0x1f); + let fraction = f64::from(bits & 0x3ff); + // 2^(e - 25) and 2^-24 as double exponents. + let scale = |e: u64| f64::from_bits((1023 + e - 25) << 52); + match exponent { + 0 => sign * fraction * scale(1), + 31 if fraction == 0.0 => sign * f64::INFINITY, + 31 => f64::NAN, + e => sign * (1024.0 + fraction) * scale(e), + } +} + +/// A vector modified immediate, spelled from the encoding the decoder kept +/// (`form`, `imm8`, `combine`, `q`); `low` is only read for the 64-bit `movi` +/// and the `fmov` forms, whose operand is the expanded pattern. +fn immediate(f: &mut Formatter<'_>, imm: SimdImmediate) -> fmt::Result { + let SimdImmediate { + rd, + imm8, + form, + combine, + q, + low, + .. + } = imm; + let name = match combine { + ImmCombine::Set => "movi", + ImmCombine::SetInverted => "mvni", + ImmCombine::Or => "orr", + ImmCombine::AndNot => "bic", + }; + match form { + ImmForm::Byte => lane_form(f, name, rd, if q { "16b" } else { "8b" }, imm8, "lsl", 0), + ImmForm::Half { shift } => { + lane_form(f, name, rd, if q { "8h" } else { "4h" }, imm8, "lsl", shift) + } + ImmForm::Word { shift } => { + lane_form(f, name, rd, if q { "4s" } else { "2s" }, imm8, "lsl", shift) + } + ImmForm::WordMask { shift } => { + lane_form(f, name, rd, if q { "4s" } else { "2s" }, imm8, "msl", shift) + } + // The 64-bit lane: the whole expanded constant, as a scalar when `Q` is clear. + ImmForm::Double if q => write!(f, "{name} v{rd}.2d, #{low:#x}"), + ImmForm::Double => write!(f, "{name} d{rd}, #{low:#x}"), + ImmForm::FmovHalf => { + write!(f, "fmov v{rd}.{}, #", if q { "8h" } else { "4h" })?; + float_immediate(f, half_to_f64(low as u16)) + } + ImmForm::FmovSingle => { + write!(f, "fmov v{rd}.{}, #", if q { "4s" } else { "2s" })?; + float_immediate(f, f64::from(f32::from_bits(low as u32))) + } + ImmForm::FmovDouble => { + write!(f, "fmov v{rd}.2d, #")?; + float_immediate(f, f64::from_bits(low)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::decode::{ImmCombine, SimdAlu, SimdDupElement, SimdExtract, SimdInsert, decode}; + + struct Show<'a>(&'a Instruction); + + impl core::fmt::Display for Show<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write( + f, + &Context { + pc: None, + symbols: &(), + }, + self.0, + ) + } + } + + /// The rendered text, kept in a fixed buffer. + struct Rendered(Text); + + impl Rendered { + fn as_str(&self) -> &str { + core::str::from_utf8(&self.0.bytes[..self.0.len]).unwrap() + } + } + + impl PartialEq<&str> for Rendered { + fn eq(&self, other: &&str) -> bool { + self.as_str() == *other + } + } + + impl core::fmt::Debug for Rendered { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.as_str()) + } + } + + fn text(instruction: &Instruction) -> Rendered { + let mut text = Text { + bytes: [0; 64], + len: 0, + }; + write!(text, "{}", Show(instruction)).unwrap(); + Rendered(text) + } + + /// Words and the `objdump` text (binutils 2.46) for them. Excluded on purpose: + /// `movi v0.16b, #0xff`, `movi d31, #0xffffffffffffffff` and the other + /// encodings that expand to the same pattern as a form listed here (`mvni v0.4s, + /// #0`); the decoder keeps only the expanded pattern, so it cannot tell them apart. + #[test] + fn matches_objdump() { + let cases: [(u32, &str); 247] = [ + (0x4f000400, "movi v0.4s, #0x0"), + (0x0f000401, "movi v1.2s, #0x0"), + (0x4f05657f, "movi v31.4s, #0xab, lsl #24"), + (0x0f002420, "movi v0.2s, #0x1, lsl #8"), + (0x4f0347e0, "movi v0.4s, #0x7f, lsl #16"), + (0x6f0747e0, "mvni v0.4s, #0xff, lsl #16"), + (0x2f002423, "mvni v3.2s, #0x1, lsl #8"), + (0x6f000423, "mvni v3.4s, #0x1"), + (0x6f000400, "mvni v0.4s, #0x0"), + (0x2f000400, "mvni v0.2s, #0x0"), + (0x4f008640, "movi v0.8h, #0x12"), + (0x0f00a420, "movi v0.4h, #0x1, lsl #8"), + (0x6f00a460, "mvni v0.8h, #0x3, lsl #8"), + (0x2f008460, "mvni v0.4h, #0x3"), + (0x4f00c420, "movi v0.4s, #0x1, msl #8"), + (0x0f07d7e0, "movi v0.2s, #0xff, msl #16"), + (0x6f00d420, "mvni v0.4s, #0x1, msl #16"), + (0x2f00c4e0, "mvni v0.2s, #0x7, msl #8"), + (0x4f00e640, "movi v0.16b, #0x12"), + (0x0f00e640, "movi v0.8b, #0x12"), + (0x2f05e4a0, "movi d0, #0xff00ff0000ff00ff"), + (0x6f05e4a0, "movi v0.2d, #0xff00ff0000ff00ff"), + (0x6f05e600, "movi v0.2d, #0xff00ffff00000000"), + (0x4f03f600, "fmov v0.4s, #1.000000000000000000e+00"), + (0x0f04f400, "fmov v0.2s, #-2.000000000000000000e+00"), + (0x6f01f7e0, "fmov v0.2d, #3.100000000000000000e+01"), + (0x6f06f400, "fmov v0.2d, #-1.250000000000000000e-01"), + (0x4f03f401, "fmov v1.4s, #5.000000000000000000e-01"), + (0x4f03f7e1, "fmov v1.4s, #1.937500000000000000e+00"), + (0x6f07f7e1, "fmov v1.2d, #-1.937500000000000000e+00"), + (0x0f02f401, "fmov v1.2s, #1.250000000000000000e-01"), + (0x4f001420, "orr v0.4s, #0x1"), + (0x4f0777e0, "orr v0.4s, #0xff, lsl #24"), + (0x0f0054e0, "orr v0.2s, #0x7, lsl #16"), + (0x4f00b420, "orr v0.8h, #0x1, lsl #8"), + (0x0f049400, "orr v0.4h, #0x80"), + (0x2f005440, "bic v0.2s, #0x2, lsl #16"), + (0x2f0094e0, "bic v0.4h, #0x7"), + (0x6f00361f, "bic v31.4s, #0x10, lsl #8"), + (0x4e209801, "cmeq v1.16b, v0.16b, #0"), + (0x0e209801, "cmeq v1.8b, v0.8b, #0"), + (0x0ea09be1, "cmeq v1.2s, v31.2s, #0"), + (0x4ee0981f, "cmeq v31.2d, v0.2d, #0"), + (0x4e609801, "cmeq v1.8h, v0.8h, #0"), + (0x4e040c20, "dup v0.4s, w1"), + (0x4e080c20, "dup v0.2d, x1"), + (0x0e010fe0, "dup v0.8b, wzr"), + (0x4e010c60, "dup v0.16b, w3"), + (0x0e020c40, "dup v0.4h, w2"), + (0x0e040fff, "dup v31.2s, wzr"), + (0x4e140420, "dup v0.4s, v1.s[2]"), + (0x4e1f0420, "dup v0.16b, v1.b[15]"), + (0x4e1807e0, "dup v0.2d, v31.d[1]"), + (0x4e1e0420, "dup v0.8h, v1.h[7]"), + (0x0e040420, "dup v0.2s, v1.s[0]"), + (0x0e0e0420, "dup v0.4h, v1.h[3]"), + (0x4e0c1c20, "mov v0.s[1], w1"), + (0x4e181fe0, "mov v0.d[1], xzr"), + (0x4e1f1c40, "mov v0.b[15], w2"), + (0x4e021fe0, "mov v0.h[0], wzr"), + (0x4e081fdf, "mov v31.d[0], x30"), + (0x6e0c4420, "mov v0.s[1], v1.s[2]"), + (0x6e1f07e0, "mov v0.b[15], v31.b[0]"), + (0x6e180420, "mov v0.d[1], v1.d[0]"), + (0x6e1e3420, "mov v0.h[7], v1.h[3]"), + (0x6e1c6420, "mov v0.s[3], v1.s[3]"), + (0x0e073c20, "umov w0, v1.b[3]"), + (0x0e1e3c20, "umov w0, v1.h[7]"), + (0x0e1c3c20, "mov w0, v1.s[3]"), + (0x4e183c20, "mov x0, v1.d[1]"), + (0x0e043ffe, "mov w30, v31.s[0]"), + (0x0e1f3fe0, "umov w0, v31.b[15]"), + (0x6e021820, "ext v0.16b, v1.16b, v2.16b, #3"), + (0x6e027820, "ext v0.16b, v1.16b, v2.16b, #15"), + (0x6e1f03ff, "ext v31.16b, v31.16b, v31.16b, #0"), + (0x4e228420, "add v0.16b, v1.16b, v2.16b"), + (0x4ee28420, "add v0.2d, v1.2d, v2.2d"), + (0x0e628420, "add v0.4h, v1.4h, v2.4h"), + (0x6e628420, "sub v0.8h, v1.8h, v2.8h"), + (0x4ea29c20, "mul v0.4s, v1.4s, v2.4s"), + (0x0ea29420, "mla v0.2s, v1.2s, v2.2s"), + (0x6e229420, "mls v0.16b, v1.16b, v2.16b"), + (0x4e221c20, "and v0.16b, v1.16b, v2.16b"), + (0x0e221c20, "and v0.8b, v1.8b, v2.8b"), + (0x4e621c20, "bic v0.16b, v1.16b, v2.16b"), + (0x4ea21c20, "orr v0.16b, v1.16b, v2.16b"), + (0x0ea21c20, "orr v0.8b, v1.8b, v2.8b"), + (0x4ea11c20, "mov v0.16b, v1.16b"), + (0x0ebf1fff, "mov v31.8b, v31.8b"), + (0x4ee21c20, "orn v0.16b, v1.16b, v2.16b"), + (0x6e221c20, "eor v0.16b, v1.16b, v2.16b"), + (0x6e621c20, "bsl v0.16b, v1.16b, v2.16b"), + (0x2ea21c20, "bit v0.8b, v1.8b, v2.8b"), + (0x6ee21c20, "bif v0.16b, v1.16b, v2.16b"), + (0x6ea28c20, "cmeq v0.4s, v1.4s, v2.4s"), + (0x4ee23420, "cmgt v0.2d, v1.2d, v2.2d"), + (0x4e623c20, "cmge v0.8h, v1.8h, v2.8h"), + (0x6e223420, "cmhi v0.16b, v1.16b, v2.16b"), + (0x6ea23c20, "cmhs v0.4s, v1.4s, v2.4s"), + (0x0ea28c20, "cmtst v0.2s, v1.2s, v2.2s"), + (0x6e226c20, "umin v0.16b, v1.16b, v2.16b"), + (0x6e626420, "umax v0.8h, v1.8h, v2.8h"), + (0x4ea26c20, "smin v0.4s, v1.4s, v2.4s"), + (0x0e226420, "smax v0.8b, v1.8b, v2.8b"), + (0x6e227420, "uabd v0.16b, v1.16b, v2.16b"), + (0x0e627420, "sabd v0.4h, v1.4h, v2.4h"), + (0x4ee20c20, "sqadd v0.2d, v1.2d, v2.2d"), + (0x6e220c20, "uqadd v0.16b, v1.16b, v2.16b"), + (0x4e622c20, "sqsub v0.8h, v1.8h, v2.8h"), + (0x6ea22c20, "uqsub v0.4s, v1.4s, v2.4s"), + (0x4e220420, "shadd v0.16b, v1.16b, v2.16b"), + (0x6e620420, "uhadd v0.8h, v1.8h, v2.8h"), + (0x4ea22420, "shsub v0.4s, v1.4s, v2.4s"), + (0x2e222420, "uhsub v0.8b, v1.8b, v2.8b"), + (0x4e221420, "srhadd v0.16b, v1.16b, v2.16b"), + (0x6e621420, "urhadd v0.8h, v1.8h, v2.8h"), + (0x4ee24420, "sshl v0.2d, v1.2d, v2.2d"), + (0x6e224420, "ushl v0.16b, v1.16b, v2.16b"), + (0x4ea24c20, "sqshl v0.4s, v1.4s, v2.4s"), + (0x6e624c20, "uqshl v0.8h, v1.8h, v2.8h"), + (0x4e225c20, "sqrshl v0.16b, v1.16b, v2.16b"), + (0x6ee25c20, "uqrshl v0.2d, v1.2d, v2.2d"), + (0x4ea2bc20, "addp v0.4s, v1.4s, v2.4s"), + (0x4e22bc20, "addp v0.16b, v1.16b, v2.16b"), + (0x6e62a420, "umaxp v0.8h, v1.8h, v2.8h"), + (0x4ea2a420, "smaxp v0.4s, v1.4s, v2.4s"), + (0x6e22ac20, "uminp v0.16b, v1.16b, v2.16b"), + (0x0ea2ac20, "sminp v0.2s, v1.2s, v2.2s"), + (0x4e31b820, "addv b0, v1.16b"), + (0x0e71b83f, "addv h31, v1.4h"), + (0x4eb1bbe0, "addv s0, v31.4s"), + (0x0e31b820, "addv b0, v1.8b"), + (0x6e71a820, "uminv h0, v1.8h"), + (0x6e30a820, "umaxv b0, v1.16b"), + (0x6eb0a820, "umaxv s0, v1.4s"), + (0x6e303820, "uaddlv h0, v1.16b"), + (0x6e703820, "uaddlv s0, v1.8h"), + (0x6eb03820, "uaddlv d0, v1.4s"), + (0x2e303820, "uaddlv h0, v1.8b"), + (0x4e030020, "tbl v0.16b, {v1.16b}, v3.16b"), + (0x4e032020, "tbl v0.16b, {v1.16b-v2.16b}, v3.16b"), + (0x4e044020, "tbl v0.16b, {v1.16b-v3.16b}, v4.16b"), + (0x0e1f6020, "tbl v0.8b, {v1.16b-v4.16b}, v31.8b"), + (0x4e0323e0, "tbl v0.16b, {v31.16b-v0.16b}, v3.16b"), + (0x4e0373c0, "tbx v0.16b, {v30.16b-v1.16b}, v3.16b"), + (0x0e0053bf, "tbx v31.8b, {v29.16b-v31.16b}, v0.8b"), + (0x0e031020, "tbx v0.8b, {v1.16b}, v3.8b"), + (0x4e021820, "uzp1 v0.16b, v1.16b, v2.16b"), + (0x4e425820, "uzp2 v0.8h, v1.8h, v2.8h"), + (0x4e823820, "zip1 v0.4s, v1.4s, v2.4s"), + (0x4ec27820, "zip2 v0.2d, v1.2d, v2.2d"), + (0x0e022820, "trn1 v0.8b, v1.8b, v2.8b"), + (0x0e826820, "trn2 v0.2s, v1.2s, v2.2s"), + (0x4e201820, "rev16 v0.16b, v1.16b"), + (0x6e600820, "rev32 v0.8h, v1.8h"), + (0x2e200820, "rev32 v0.8b, v1.8b"), + (0x4ea00820, "rev64 v0.4s, v1.4s"), + (0x4e200820, "rev64 v0.16b, v1.16b"), + (0x4e205820, "cnt v0.16b, v1.16b"), + (0x0e205820, "cnt v0.8b, v1.8b"), + (0x6e205820, "mvn v0.16b, v1.16b"), + (0x2e205820, "mvn v0.8b, v1.8b"), + (0x6e605820, "rbit v0.16b, v1.16b"), + (0x2e605820, "rbit v0.8b, v1.8b"), + (0x4ea08820, "cmgt v0.4s, v1.4s, #0"), + (0x6e208820, "cmge v0.16b, v1.16b, #0"), + (0x6e609820, "cmle v0.8h, v1.8h, #0"), + (0x4ee0a820, "cmlt v0.2d, v1.2d, #0"), + (0x0ea0a820, "cmlt v0.2s, v1.2s, #0"), + (0x4ea0b820, "abs v0.4s, v1.4s"), + (0x4ee0b820, "abs v0.2d, v1.2d"), + (0x6e20b820, "neg v0.16b, v1.16b"), + (0x2e60b820, "neg v0.4h, v1.4h"), + (0x0e212820, "xtn v0.8b, v1.8h"), + (0x4e212820, "xtn2 v0.16b, v1.8h"), + (0x0e612820, "xtn v0.4h, v1.4s"), + (0x4ea12820, "xtn2 v0.4s, v1.2d"), + (0x0ea14820, "sqxtn v0.2s, v1.2d"), + (0x4e614820, "sqxtn2 v0.8h, v1.4s"), + (0x2e214820, "uqxtn v0.8b, v1.8h"), + (0x6ea14820, "uqxtn2 v0.4s, v1.2d"), + (0x2e612820, "sqxtun v0.4h, v1.4s"), + (0x6e212820, "sqxtun2 v0.16b, v1.8h"), + (0x4e202820, "saddlp v0.8h, v1.16b"), + (0x0e202820, "saddlp v0.4h, v1.8b"), + (0x6e602820, "uaddlp v0.4s, v1.8h"), + (0x2ea02820, "uaddlp v0.1d, v1.2s"), + (0x6ea02820, "uaddlp v0.2d, v1.4s"), + (0x4e606820, "sadalp v0.4s, v1.8h"), + (0x6e206820, "uadalp v0.8h, v1.16b"), + (0x0e220020, "saddl v0.8h, v1.8b, v2.8b"), + (0x4e220020, "saddl2 v0.8h, v1.16b, v2.16b"), + (0x2e620020, "uaddl v0.4s, v1.4h, v2.4h"), + (0x6ea20020, "uaddl2 v0.2d, v1.4s, v2.4s"), + (0x0ea22020, "ssubl v0.2d, v1.2s, v2.2s"), + (0x6e622020, "usubl2 v0.4s, v1.8h, v2.8h"), + (0x0e221020, "saddw v0.8h, v1.8h, v2.8b"), + (0x4ea21020, "saddw2 v0.2d, v1.2d, v2.4s"), + (0x2e621020, "uaddw v0.4s, v1.4s, v2.4h"), + (0x6e221020, "uaddw2 v0.8h, v1.8h, v2.16b"), + (0x0ea23020, "ssubw v0.2d, v1.2d, v2.2s"), + (0x6e623020, "usubw2 v0.4s, v1.4s, v2.8h"), + (0x0e227020, "sabdl v0.8h, v1.8b, v2.8b"), + (0x6e627020, "uabdl2 v0.4s, v1.8h, v2.8h"), + (0x0e628020, "smlal v0.4s, v1.4h, v2.4h"), + (0x6ea28020, "umlal2 v0.2d, v1.4s, v2.4s"), + (0x0e22a020, "smlsl v0.8h, v1.8b, v2.8b"), + (0x6e62a020, "umlsl2 v0.4s, v1.8h, v2.8h"), + (0x0ea2c020, "smull v0.2d, v1.2s, v2.2s"), + (0x6e22c020, "umull2 v0.8h, v1.16b, v2.16b"), + (0x5ef1b820, "addp d0, v1.2d"), + (0x5e070420, "mov b0, v1.b[3]"), + (0x5e1e043f, "mov h31, v1.h[7]"), + (0x5e140420, "mov s0, v1.s[2]"), + (0x5e1807e0, "mov d0, v31.d[1]"), + (0x5ee28420, "add d0, d1, d2"), + (0x7ee28420, "sub d0, d1, d2"), + (0x5ee23420, "cmgt d0, d1, d2"), + (0x7ee23420, "cmhi d0, d1, d2"), + (0x5ee23c20, "cmge d0, d1, d2"), + (0x7ee23c20, "cmhs d0, d1, d2"), + (0x7ee28c20, "cmeq d0, d1, d2"), + (0x5ee28c3f, "cmtst d31, d1, d2"), + (0x4f205420, "shl v0.4s, v1.4s, #0"), + (0x4f0f5420, "shl v0.16b, v1.16b, #7"), + (0x4f7f5420, "shl v0.2d, v1.2d, #63"), + (0x0f1f5420, "shl v0.4h, v1.4h, #15"), + (0x0f080420, "sshr v0.8b, v1.8b, #8"), + (0x4f400420, "sshr v0.2d, v1.2d, #64"), + (0x6f3f0420, "ushr v0.4s, v1.4s, #1"), + (0x6f100420, "ushr v0.8h, v1.8h, #16"), + (0x0f0fa420, "sshll v0.8h, v1.8b, #7"), + (0x4f1fa420, "sshll2 v0.4s, v1.8h, #15"), + (0x0f3fa420, "sshll v0.2d, v1.2s, #31"), + (0x2f09a420, "ushll v0.8h, v1.8b, #1"), + (0x6f21a420, "ushll2 v0.2d, v1.4s, #1"), + (0x0f08a420, "sxtl v0.8h, v1.8b"), + (0x4f10a420, "sxtl2 v0.4s, v1.8h"), + (0x0f20a420, "sxtl v0.2d, v1.2s"), + (0x2f08a420, "uxtl v0.8h, v1.8b"), + (0x6f20a420, "uxtl2 v0.2d, v1.4s"), + (0x2f10a420, "uxtl v0.4s, v1.4h"), + (0x0f088420, "shrn v0.8b, v1.8h, #8"), + (0x4f0f8420, "shrn2 v0.16b, v1.8h, #1"), + (0x0f208420, "shrn v0.2s, v1.2d, #32"), + (0x4f3b8420, "shrn2 v0.4s, v1.2d, #5"), + (0x0f108420, "shrn v0.4h, v1.4s, #16"), + ]; + for (word, expected) in cases { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:08x}: {e:?}")); + assert_eq!(text(&instruction), expected, "{word:08x}"); + } + } + + /// Words verified against GNU `objdump`. + #[test] + fn vector_moves_match_objdump() { + for (word, expected) in [ + (0x2efe1fe0, "bif v0.8b, v31.8b, v30.8b"), + (0x0ee01c20, "orn v0.8b, v1.8b, v0.8b"), + (0x0ea21c20, "orr v0.8b, v1.8b, v2.8b"), + (0x0e211c20, "and v0.8b, v1.8b, v1.8b"), + (0x4e1c1c20, "mov v0.s[3], w1"), + (0x4e083c20, "mov x0, v1.d[0]"), + (0x0e013c20, "umov w0, v1.b[0]"), + ] { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:08x}: {e:?}")); + assert_eq!(text(&instruction), expected, "{word:08x}"); + } + } + + /// Words checked against GNU `objdump`: the modified-immediate forms keep the + /// arrangement and shift, so encodings that expand to the same pattern differ. + #[test] + fn modified_immediates_keep_their_encoding() { + let cases: [(u32, &str); 27] = [ + (0x4f000400, "movi v0.4s, #0x0"), + (0x0f000400, "movi v0.2s, #0x0"), + (0x6f00e401, "movi v1.2d, #0x0"), + (0x2f00e400, "movi d0, #0x0"), + (0x4f00e400, "movi v0.16b, #0x0"), + (0x0f00e460, "movi v0.8b, #0x3"), + (0x6f0707ff, "mvni v31.4s, #0xff"), + (0x6f00045f, "mvni v31.4s, #0x2"), + (0x6f00c400, "mvni v0.4s, #0x0, msl #8"), + (0x4f00d420, "movi v0.4s, #0x1, msl #16"), + (0x0f008420, "movi v0.4h, #0x1"), + (0x4f00a420, "movi v0.8h, #0x1, lsl #8"), + (0x6f00a420, "mvni v0.8h, #0x1, lsl #8"), + (0x4f009420, "orr v0.8h, #0x1"), + (0x4f00b420, "orr v0.8h, #0x1, lsl #8"), + (0x2f00b420, "bic v0.4h, #0x1, lsl #8"), + (0x0f00540a, "orr v10.2s, #0x0, lsl #16"), + (0x0f006400, "movi v0.2s, #0x0, lsl #24"), + (0x4f00f400, "fmov v0.4s, #2.000000000000000000e+00"), + (0x6f00f400, "fmov v0.2d, #2.000000000000000000e+00"), + (0x4f00fc00, "fmov v0.8h, #2.000000000000000000e+00"), + (0x0f00fc00, "fmov v0.4h, #2.000000000000000000e+00"), + (0x4f04fc80, "fmov v0.8h, #-2.500000000000000000e+00"), + (0x2f00e420, "movi d0, #0xff"), + (0x2f0367e1, "mvni v1.2s, #0x7f, lsl #24"), + (0x2e023820, "ext v0.8b, v1.8b, v2.8b, #7"), + (0x6e024820, "ext v0.16b, v1.16b, v2.16b, #9"), + ]; + for (word, expected) in cases { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:08x}: {e:?}")); + assert_eq!(text(&instruction), expected, "{word:08x}"); + } + // Not instructions (`objdump`: `.inst ... ; undefined`) and so not decoded: an + // `ext .8b` index above 7, a modified immediate with bit 11 set outside the + // half-precision `fmov`, `fmov .2d` with `Q` clear. + for word in [0x2e164372, 0x0f000ced, 0x4f00af48, 0x2f00f420, 0x6f00fc20] { + assert!(decode(word).is_err(), "{word:08x}"); + } + } + + /// Fields beyond what the decoder produces must still print, not panic. + #[test] + fn extreme_fields_do_not_panic() { + use crate::simd::Op; + let ops = [ + Op::ShiftImm { + right: true, + signed: true, + size: u32::MAX, + q: true, + shift: u32::MAX, + rd: 255, + rn: 255, + }, + Op::ShiftLong { + signed: false, + size: 7, + shift: 1, + upper: true, + rd: 255, + rn: 255, + }, + Op::ScalarDup { + size: 99, + index: u32::MAX, + rd: 255, + rn: 255, + }, + Op::Narrow { + kind: crate::simd::Narrow::Plain, + size: 3, + upper: true, + rd: 0, + rn: 0, + }, + ]; + for op in ops { + let _ = text(&Instruction::Simd(op)); + } + let _ = text(&Instruction::SimdAlu(SimdAlu { + op: SimdAluOp::UAddLV, + rd: 255, + rn: 255, + rm: 255, + size: 255, + q: true, + })); + let _ = text(&Instruction::SimdDupElement(SimdDupElement { + rd: 255, + rn: 255, + esize: 255, + index: 255, + q: false, + })); + let _ = text(&Instruction::SimdInsert(SimdInsert { + rd: 255, + rn: 255, + esize: 255, + index: 255, + })); + let _ = text(&Instruction::SimdExtract(SimdExtract { + rd: 255, + rn: 255, + esize: 255, + index: 255, + })); + let _ = text(&Instruction::SimdTable(SimdTable { + rd: 255, + rn: 255, + rm: 255, + len: 255, + extend: true, + q: true, + })); + for low in [0, 1, u64::MAX, 0x0123_4567_89ab_cdef, 0x8000_0000_8000_0000] { + for combine in [ + ImmCombine::Set, + ImmCombine::SetInverted, + ImmCombine::Or, + ImmCombine::AndNot, + ] { + for form in [ + ImmForm::Byte, + ImmForm::Half { shift: 255 }, + ImmForm::Word { shift: 255 }, + ImmForm::WordMask { shift: 255 }, + ImmForm::Double, + ImmForm::FmovHalf, + ImmForm::FmovSingle, + ImmForm::FmovDouble, + ] { + for q in [false, true] { + let _ = text(&Instruction::SimdImm(SimdImmediate { + rd: 255, + imm8: 255, + form, + low, + high: if q { low } else { 0 }, + combine, + q, + })); + } + } + } + } + } +} diff --git a/src/isa/aarch64/src/fp.rs b/src/isa/aarch64/src/fp.rs new file mode 100644 index 0000000..3e21b4b --- /dev/null +++ b/src/isa/aarch64/src/fp.rs @@ -0,0 +1,358 @@ +//! Scalar floating point (single and double precision) and the vector `fabs`/`fneg` +//! forms: the decoded shape of every word [`decode`] recognises. +//! +//! Half precision is not decoded here (`type` 3), nor is anything with a reserved +//! `type`; those words are not recognised. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Precision { + Single, + Double, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Unary { + Move, + Abs, + Neg, + Sqrt, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Binary { + Mul, + Div, + Add, + Sub, + Max, + Min, + MaxNum, + MinNum, + Nmul, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Fused { + Madd, + Msub, + Nmadd, + Nmsub, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + Unary { + kind: Unary, + p: Precision, + rd: u8, + rn: u8, + }, + /// `fcvt`: between single and double; `p` is the source precision. + Convert { + p: Precision, + rd: u8, + rn: u8, + }, + Binary { + kind: Binary, + p: Precision, + rd: u8, + rn: u8, + rm: u8, + }, + Fused { + kind: Fused, + p: Precision, + rd: u8, + rn: u8, + rm: u8, + ra: u8, + }, + /// `fcmp`/`fcmpe`; no `rm` compares with zero. `signaling` is `fcmpe`. + Compare { + p: Precision, + rn: u8, + rm: Option, + signaling: bool, + }, + CondCompare { + p: Precision, + rn: u8, + rm: u8, + cond: u32, + nzcv: u32, + }, + Select { + p: Precision, + rd: u8, + rn: u8, + rm: u8, + cond: u32, + }, + Immediate { + p: Precision, + rd: u8, + imm8: u64, + }, + /// `scvtf`/`ucvtf` from a general register. + IntToFloat { + signed: bool, + p: Precision, + wide: bool, + rd: u8, + rn: u8, + }, + /// `fcvtzs`/`fcvtzu` to a general register. + FloatToInt { + signed: bool, + p: Precision, + wide: bool, + rd: u8, + rn: u8, + }, + /// `scvtf`/`ucvtf` between SIMD&FP registers (the source holds an integer). + ScalarIntToFloat { + signed: bool, + p: Precision, + rd: u8, + rn: u8, + }, + /// `fcvtzs`/`fcvtzu` between SIMD&FP registers. + ScalarFloatToInt { + signed: bool, + p: Precision, + rd: u8, + rn: u8, + }, + /// Vector `fabs`/`fneg`. + VectorSign { + negate: bool, + double: bool, + q: bool, + rd: u8, + rn: u8, + }, +} + +fn precision(ty: u32) -> Option { + match ty { + 0 => Some(Precision::Single), + 1 => Some(Precision::Double), + // 3 is half precision, which is not modelled; 2 is reserved. + _ => None, + } +} + +/// Decode `word` as a floating-point instruction, or `None` when it is not one. +pub fn decode(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = ((word >> 16) & 31) as u8; + let ty = (word >> 22) & 3; + // Scalar floating point: `M=0 S=0 01111 0 type 1 ...`. + if word & 0xff20_0c00 == 0x1e20_0800 { + let kind = match (word >> 12) & 15 { + 0 => Binary::Mul, + 1 => Binary::Div, + 2 => Binary::Add, + 3 => Binary::Sub, + 4 => Binary::Max, + 5 => Binary::Min, + 6 => Binary::MaxNum, + 7 => Binary::MinNum, + 8 => Binary::Nmul, + _ => return None, + }; + return Some(Op::Binary { + kind, + p: precision(ty)?, + rd, + rn, + rm, + }); + } + // One-source: bits 14:10 = 10000; the opcode (bits 20:15) includes bit 15. + if word & 0xff20_7c00 == 0x1e20_4000 { + let p = precision(ty)?; + let kind = match (word >> 15) & 0x3f { + 0 => Unary::Move, + 1 => Unary::Abs, + 2 => Unary::Neg, + 3 => Unary::Sqrt, + // `fcvt` to single from double, to double from single. + 4 if p == Precision::Double => return Some(Op::Convert { p, rd, rn }), + 5 if p == Precision::Single => return Some(Op::Convert { p, rd, rn }), + _ => return None, + }; + return Some(Op::Unary { kind, p, rd, rn }); + } + if word & 0xff20_fc07 == 0x1e20_2000 { + let zero = word & 8 != 0; + return Some(Op::Compare { + p: precision(ty)?, + rn, + rm: (!zero).then_some(rm), + signaling: word & 0x10 != 0, + }); + } + if word & 0xff20_0c10 == 0x1e20_0400 { + return Some(Op::CondCompare { + p: precision(ty)?, + rn, + rm, + cond: (word >> 12) & 15, + nzcv: word & 15, + }); + } + if word & 0xff20_0c00 == 0x1e20_0c00 { + return Some(Op::Select { + p: precision(ty)?, + rd, + rn, + rm, + cond: (word >> 12) & 15, + }); + } + if word & 0xff20_1fe0 == 0x1e20_1000 { + return Some(Op::Immediate { + p: precision(ty)?, + rd, + imm8: u64::from((word >> 13) & 0xff), + }); + } + if word & 0xff00_0000 == 0x1f00_0000 { + let kind = match ((word >> 21) & 1, (word >> 15) & 1) { + (0, 0) => Fused::Madd, + (0, _) => Fused::Msub, + (_, 0) => Fused::Nmadd, + _ => Fused::Nmsub, + }; + return Some(Op::Fused { + kind, + p: precision(ty)?, + rd, + rn, + rm, + ra: ((word >> 10) & 31) as u8, + }); + } + // Conversions between floating point and general registers. The `fmov` forms + // (rmode 0, opcode 6 and 7) are `SimdFmov`. + if word & 0x7f20_fc00 == 0x1e20_0000 { + let (p, wide) = (precision(ty)?, word & 0x8000_0000 != 0); + return match ((word >> 19) & 3, (word >> 16) & 7) { + (0, 2) => Some(Op::IntToFloat { + signed: true, + p, + wide, + rd, + rn, + }), + (0, 3) => Some(Op::IntToFloat { + signed: false, + p, + wide, + rd, + rn, + }), + (3, 0) => Some(Op::FloatToInt { + signed: true, + p, + wide, + rd, + rn, + }), + (3, 1) => Some(Op::FloatToInt { + signed: false, + p, + wide, + rd, + rn, + }), + _ => None, + }; + } + // The same conversions between SIMD&FP registers: bit 22 is `sz`. + let p = if word & 0x0040_0000 != 0 { + Precision::Double + } else { + Precision::Single + }; + match word & 0xffbf_fc00 { + 0x5e21_d800 => { + return Some(Op::ScalarIntToFloat { + signed: true, + p, + rd, + rn, + }); + } + 0x7e21_d800 => { + return Some(Op::ScalarIntToFloat { + signed: false, + p, + rd, + rn, + }); + } + 0x5ea1_b800 => { + return Some(Op::ScalarFloatToInt { + signed: true, + p, + rd, + rn, + }); + } + 0x7ea1_b800 => { + return Some(Op::ScalarFloatToInt { + signed: false, + p, + rd, + rn, + }); + } + _ => {} + } + // Vector `fabs` (U=0) and `fneg` (U=1). + if word & 0x9fbf_fc00 == 0x0ea0_f800 { + let (q, double) = (word & 0x4000_0000 != 0, word & 0x0040_0000 != 0); + // `.1d` is reserved. + if double && !q { + return None; + } + return Some(Op::VectorSign { + negate: word & 0x2000_0000 != 0, + double, + q, + rd, + rn, + }); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_encodings_are_not_claimed() { + // fmov d0, x1 and fmov x0, d1 are `SimdFmov`, not conversions. + assert_eq!(decode(0x9e670020), None); + assert_eq!(decode(0x9e660020), None); + // Half precision (type 3) and reserved type 2. + assert_eq!(decode(0x1ee22820), None); + assert_eq!(decode(0x1ea22820), None); + // fneg v0.1d is reserved. + assert_eq!(decode(0x2ee0f820), None); + assert!(decode(0x1e620820).is_some()); + } + + #[test] + fn compare_signaling() { + let cmp = |word| match decode(word) { + Some(Op::Compare { rm, signaling, .. }) => (rm, signaling), + other => panic!("{other:?}"), + }; + assert_eq!(cmp(0x1e222020), (Some(2), false)); // fcmp s1, s2 + assert_eq!(cmp(0x1e222030), (Some(2), true)); // fcmpe s1, s2 + assert_eq!(cmp(0x1e602008), (None, false)); // fcmp d0, #0.0 + assert_eq!(cmp(0x1e202038), (None, true)); // fcmpe s1, #0.0 + } +} diff --git a/src/isa/aarch64/src/gxf.rs b/src/isa/aarch64/src/gxf.rs new file mode 100644 index 0000000..a17d32d --- /dev/null +++ b/src/isa/aarch64/src/gxf.rs @@ -0,0 +1,41 @@ +//! Guarded execution (GXF): `genter` and `gexit`, the Apple instructions that move +//! between EL and the guarded levels GL1 and GL2 (Asahi Linux documentation, "Apple +//! Proprietary Instructions" and "SPRR and GXF"). + +/// `gexit`: leave guarded mode. +const GEXIT: u32 = 0x0020_1400; +/// `genter` with its `imm5` in bits 4:0 (stored in `ESR_GLx[5:0]`). +const GENTER: u32 = 0x0020_1420; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + Gexit, + Genter { imm5: u8 }, +} + +/// Decode `word` as `genter` or `gexit`, or `None` when it is neither. +pub fn decode(word: u32) -> Option { + if word == GEXIT { + Some(Op::Gexit) + } else if word & !0x1f == GENTER { + Some(Op::Genter { + imm5: (word & 0x1f) as u8, + }) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn genter_and_gexit_are_recognised_with_any_genter_immediate() { + assert_eq!(decode(GEXIT), Some(Op::Gexit)); + assert_eq!(decode(GENTER), Some(Op::Genter { imm5: 0 })); + assert_eq!(decode(GENTER | 0x1f), Some(Op::Genter { imm5: 31 })); + assert_eq!(decode(0xd503_201f), None); + assert_eq!(decode(GEXIT + 4), None); + } +} diff --git a/src/isa/aarch64/src/lib.rs b/src/isa/aarch64/src/lib.rs new file mode 100644 index 0000000..78bafde --- /dev/null +++ b/src/isa/aarch64/src/lib.rs @@ -0,0 +1,27 @@ +//! The AArch64 (A64) instruction set: what an instruction word *is*. +//! +//! This crate owns architectural knowledge and nothing else. It does not know +//! about guest state, memory, block translation, register allocation or +//! executable memory; consumers (disassembly, lifting, emulation) decode a word +//! once here and work from the result. +//! +//! * [`decode`] turns an instruction word into a structured +//! [`decode::Instruction`]. It is the only decoder: [`fp`], [`simd`], [`pauth`] +//! and [`gxf`] hold the instruction families it delegates to. +//! * [`format`] prints a decoded instruction as assembly text (`objdump` spelling). +//! * [`flow`] answers the control-flow question for analysis. +//! * [`simd_struct`] describes SIMD structure loads and stores. + +#![no_std] + +#[cfg(test)] +extern crate std; + +pub mod decode; +pub mod flow; +pub mod format; +pub mod fp; +pub mod gxf; +pub mod pauth; +pub mod simd; +pub mod simd_struct; diff --git a/src/isa/aarch64/src/pauth.rs b/src/isa/aarch64/src/pauth.rs new file mode 100644 index 0000000..174c2d5 --- /dev/null +++ b/src/isa/aarch64/src/pauth.rs @@ -0,0 +1,354 @@ +//! Pointer authentication (FEAT_PAuth) instructions: the decoded shape of every +//! word [`decode`] recognises, including the hint-space forms (`PACIASP`, ...). +//! +//! The cipher and what authentication does to a pointer are the emulator's business. + +/// The key the instruction uses. `Ga` is the generic key, used only by `PACGA`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum KeyId { + Ia, + Ib, + Da, + Db, + Ga, +} + +impl KeyId { + /// The `SCTLR_EL1` bit that enables the key, if any. The generic key has none. + pub fn enable_bit(self) -> Option { + match self { + Self::Ia => Some(31), + Self::Ib => Some(30), + Self::Da => Some(27), + Self::Db => Some(13), + Self::Ga => None, + } + } + + /// The `keynumber` the architecture reports in an authentication failure. + pub fn number(self) -> u64 { + match self { + Self::Ia => 0, + Self::Ib => 1, + Self::Da => 2, + Self::Db => 3, + Self::Ga => 4, + } + } +} + +/// The modifier operand: zero, or a register where register 31 is `SP`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Modifier { + Zero, + Reg(u8), +} + +/// One PAuth instruction, decoded. +/// +/// `hint` marks the hint-space spellings (`PACIASP`, `PACIA1716`, `PACIAZ`, `XPACLRI`, ...), +/// which execute as no-ops without PAuth; their register and modifier are fixed by the +/// opcode (`LR` signed with `SP`, `X17` with `X16`, `LR` with zero, `LR` stripped). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + /// `PACIA`, `PACIZA`, `PACDA`, ... and `PACIASP`, `PACIA1716`, `PACIAZ`, ... + Sign { + key: KeyId, + reg: u8, + modifier: Modifier, + hint: bool, + }, + /// `AUTIA`, `AUTIZA`, `AUTDA`, ... and `AUTIASP`, `AUTIA1716`, `AUTIAZ`, ... + Auth { + key: KeyId, + reg: u8, + modifier: Modifier, + hint: bool, + }, + /// `XPACI` (`data` clear), `XPACD` (`data` set) and the hint `XPACLRI`. + Strip { + data: bool, + reg: u8, + hint: bool, + }, + Pacga { + rd: u8, + rn: u8, + modifier: Modifier, + }, + /// `BRAA`, `BRAB`, `BLRAA`, `BLRAB` and their zero-modifier forms. + Branch { + key: KeyId, + link: bool, + target: u8, + modifier: Modifier, + }, + /// `RETAA` and `RETAB`: return to `LR`, authenticated with `SP`. + Return { + key: KeyId, + }, + /// `ERETAA` and `ERETAB`: return from the exception, authenticating `ELR_EL1`. + ExceptionReturn { + key: KeyId, + }, +} + +/// The hint-space forms (`PACIASP` and friends), which are no-ops without PAuth. +fn hint(word: u32) -> Option { + if word & 0xffff_f01f != 0xd503_201f { + return None; + } + let hint = (word >> 5) & 0x7f; + // Bit 2 selects authenticate over sign, bit 1 the B key over the A key. + let key = if hint & 2 != 0 { KeyId::Ib } else { KeyId::Ia }; + let (reg, modifier) = match hint { + 7 => { + return Some(Op::Strip { + data: false, + reg: 30, + hint: true, + }); + } + // PACIA1716, PACIB1716, AUTIA1716, AUTIB1716. + 8 | 10 | 12 | 14 => (17, Modifier::Reg(16)), + // PAC/AUT{IA,IB}Z (even) and PAC/AUT{IA,IB}SP (odd). + 24..=31 if hint & 1 == 0 => (30, Modifier::Zero), + 24..=31 => (30, Modifier::Reg(31)), + _ => return None, + }; + Some(if hint & 4 != 0 { + Op::Auth { + key, + reg, + modifier, + hint: true, + } + } else { + Op::Sign { + key, + reg, + modifier, + hint: true, + } + }) +} + +/// Decode `word` as a pointer-authentication instruction, or `None` when it is not one. +pub fn decode(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = rd; + // Data-processing, one source: PAC and AUT (opcodes 0-15), and XPAC (16-17). + if word & 0xffff_0000 == 0xdac1_0000 { + let opcode = (word >> 10) & 0x3f; + let key = [KeyId::Ia, KeyId::Ib, KeyId::Da, KeyId::Db][(opcode & 3) as usize]; + // The zero-modifier forms have no modifier register: Rn must be 31. + let modifier = if opcode & 8 != 0 { + if rn != 31 { + return None; + } + Modifier::Zero + } else { + Modifier::Reg(rn) + }; + return match opcode { + 0..=15 if opcode & 4 == 0 => Some(Op::Sign { + key, + reg: rd, + modifier, + hint: false, + }), + 0..=15 => Some(Op::Auth { + key, + reg: rd, + modifier, + hint: false, + }), + 16 | 17 if rn == 31 => Some(Op::Strip { + data: opcode == 17, + reg: rd, + hint: false, + }), + _ => None, + }; + } + // Data-processing, two sources: PACGA Xd, Xn, Xm|SP. + if word & 0xffe0_fc00 == 0x9ac0_3000 { + let rm = ((word >> 16) & 31) as u8; + return Some(Op::Pacga { + rd, + rn, + modifier: Modifier::Reg(rm), + }); + } + // Branches with authentication. Bit 10 selects the B key; bit 11 is always set. + let b_key = if word & 0x400 != 0 { + KeyId::Ib + } else { + KeyId::Ia + }; + if word & 0xffff_f800 == 0xd71f_0800 { + return Some(Op::Branch { + key: b_key, + link: false, + target: rn, + modifier: Modifier::Reg(rm), + }); + } + if word & 0xffff_f81f == 0xd61f_081f { + return Some(Op::Branch { + key: b_key, + link: false, + target: rn, + modifier: Modifier::Zero, + }); + } + if word & 0xffff_f800 == 0xd73f_0800 { + return Some(Op::Branch { + key: b_key, + link: true, + target: rn, + modifier: Modifier::Reg(rm), + }); + } + if word & 0xffff_f81f == 0xd63f_081f { + return Some(Op::Branch { + key: b_key, + link: true, + target: rn, + modifier: Modifier::Zero, + }); + } + if word & 0xffff_fbff == 0xd65f_0bff { + return Some(Op::Return { key: b_key }); + } + if word & 0xffff_fbff == 0xd69f_0bff { + return Some(Op::ExceptionReturn { key: b_key }); + } + hint(word) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sign(key: KeyId, reg: u8, modifier: Modifier, hint: bool) -> Option { + Some(Op::Sign { + key, + reg, + modifier, + hint, + }) + } + + fn auth(key: KeyId, reg: u8, modifier: Modifier, hint: bool) -> Option { + Some(Op::Auth { + key, + reg, + modifier, + hint, + }) + } + + #[test] + fn the_instruction_encodings_decode_as_their_names() { + // Assembled with GNU as for armv8.3-a. + assert_eq!( + decode(0xdac1_0020), + sign(KeyId::Ia, 0, Modifier::Reg(1), false) + ); + assert_eq!( + decode(0xdac1_11ac), + auth(KeyId::Ia, 12, Modifier::Reg(13), false) + ); + assert_eq!( + decode(0xdac1_23f0), + sign(KeyId::Ia, 16, Modifier::Zero, false) + ); + assert_eq!( + decode(0xdac1_43f8), + Some(Op::Strip { + data: false, + reg: 24, + hint: false + }) + ); + assert_eq!( + decode(0xd71f_0c43), + Some(Op::Branch { + key: KeyId::Ib, + link: false, + target: 2, + modifier: Modifier::Reg(3) + }) + ); + assert_eq!( + decode(0xd63f_095f), + Some(Op::Branch { + key: KeyId::Ia, + link: true, + target: 10, + modifier: Modifier::Zero + }) + ); + assert_eq!(decode(0xd65f_0fff), Some(Op::Return { key: KeyId::Ib })); + assert_eq!( + decode(0xd69f_0bff), + Some(Op::ExceptionReturn { key: KeyId::Ia }) + ); + // Plain BR and ordinary one-source data processing are not PAuth. + assert_eq!(decode(0xd61f_0000), None); + assert_eq!(decode(0xdac0_0000), None); + } + + #[test] + fn xpaci_and_xpacd_are_distinct() { + // `xpaci x0`, `xpacd x0`, `xpaci x30`, `xpacd x30`. + let strip = |data, reg, hint| Some(Op::Strip { data, reg, hint }); + assert_eq!(decode(0xdac1_43e0), strip(false, 0, false)); + assert_eq!(decode(0xdac1_47e0), strip(true, 0, false)); + assert_eq!(decode(0xdac1_43fe), strip(false, 30, false)); + assert_eq!(decode(0xdac1_47fe), strip(true, 30, false)); + // The hint `xpaclri` is its own spelling of stripping LR. + assert_eq!(decode(0xd503_20ff), strip(false, 30, true)); + } + + #[test] + fn hint_space_forms_differ_from_the_register_forms() { + use KeyId::{Ia, Ib}; + use Modifier::{Reg, Zero}; + // paciaz, paciasp, pacibz, pacibsp, autiaz, autiasp, autibz, autibsp. + assert_eq!(decode(0xd503_231f), sign(Ia, 30, Zero, true)); + assert_eq!(decode(0xd503_233f), sign(Ia, 30, Reg(31), true)); + assert_eq!(decode(0xd503_235f), sign(Ib, 30, Zero, true)); + assert_eq!(decode(0xd503_237f), sign(Ib, 30, Reg(31), true)); + assert_eq!(decode(0xd503_239f), auth(Ia, 30, Zero, true)); + assert_eq!(decode(0xd503_23bf), auth(Ia, 30, Reg(31), true)); + assert_eq!(decode(0xd503_23df), auth(Ib, 30, Zero, true)); + assert_eq!(decode(0xd503_23ff), auth(Ib, 30, Reg(31), true)); + // pacia1716, pacib1716, autia1716, autib1716. + assert_eq!(decode(0xd503_211f), sign(Ia, 17, Reg(16), true)); + assert_eq!(decode(0xd503_215f), sign(Ib, 17, Reg(16), true)); + assert_eq!(decode(0xd503_219f), auth(Ia, 17, Reg(16), true)); + assert_eq!(decode(0xd503_21df), auth(Ib, 17, Reg(16), true)); + // The register forms with the same operands: `pacia x30, sp`, `pacia x17, x16`, + // `paciza x30`, `autia x30, sp`. + assert_eq!(decode(0xdac1_03fe), sign(Ia, 30, Reg(31), false)); + assert_eq!(decode(0xdac1_0211), sign(Ia, 17, Reg(16), false)); + assert_eq!(decode(0xdac1_23fe), sign(Ia, 30, Zero, false)); + assert_eq!(decode(0xdac1_13fe), auth(Ia, 30, Reg(31), false)); + // Other hint numbers are not PAuth. + assert_eq!(decode(0xd503_203f), None); + assert_eq!(decode(0xd503_213f), None); + assert_eq!(decode(0xd503_243f), None); + } + + #[test] + fn zero_modifier_forms_need_register_31() { + // `paciza x0` with Rn = 1 is unallocated (objdump: `.inst ... ; undefined`). + assert_eq!(decode(0xdac1_2020), None); + assert_eq!(decode(0xdac1_3020), None); + // XPAC likewise needs Rn = 31. + assert_eq!(decode(0xdac1_4020), None); + } +} diff --git a/src/isa/aarch64/src/simd.rs b/src/isa/aarch64/src/simd.rs new file mode 100644 index 0000000..83b462a --- /dev/null +++ b/src/isa/aarch64/src/simd.rs @@ -0,0 +1,441 @@ +//! Integer Advanced SIMD instructions the structured decoder has no form for: the +//! vector permutes (`uzp`, `zip`, `trn`), the two-register-miscellaneous group +//! (`rev`, `cnt`, `mvn`, compares against zero, `abs`/`neg`, the narrowing `xtn` +//! family and the pairwise-long adds), the three-registers-different long and wide +//! arithmetic (`uaddl`, `umull`, ...), immediate shifts, and the 64-bit scalar forms. +//! +//! [`decode`] classifies a word; executing it is the emulator's business. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Permute { + Uzp1, + Uzp2, + Zip1, + Zip2, + Trn1, + Trn2, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Zero { + Gt, + Ge, + Le, + Lt, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Narrow { + Plain, + SignedSat, + UnsignedSat, + SignedToUnsignedSat, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Long { + Add, + AddWide, + Sub, + SubWide, + Abd, + Mul, + Mla, + Mls, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Scalar { + Add, + Sub, + Gt, + Hi, + Ge, + Hs, + Eq, + Tst, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + Permute { + kind: Permute, + size: u32, + q: bool, + rd: u8, + rn: u8, + rm: u8, + }, + /// `rev16`/`rev32`/`rev64`: reverse the lanes inside each group of `group` bits. + Reverse { + group: u32, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Count { + q: bool, + rd: u8, + rn: u8, + }, + Not { + q: bool, + rd: u8, + rn: u8, + }, + ReverseBits { + q: bool, + rd: u8, + rn: u8, + }, + CompareZero { + kind: Zero, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Abs { + negate: bool, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + /// `xtn`/`sqxtn`/`uqxtn`/`sqxtun`; `upper` is the `2` form (keeps the low half). + Narrow { + kind: Narrow, + size: u32, + upper: bool, + rd: u8, + rn: u8, + }, + /// `saddlp`/`uaddlp` and, with `accumulate`, `sadalp`/`uadalp`. + PairLong { + signed: bool, + accumulate: bool, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Long { + kind: Long, + signed: bool, + size: u32, + upper: bool, + rd: u8, + rn: u8, + rm: u8, + }, + ScalarAddp { + rd: u8, + rn: u8, + }, + ScalarDup { + size: u32, + index: u32, + rd: u8, + rn: u8, + }, + ScalarThree { + kind: Scalar, + rd: u8, + rn: u8, + rm: u8, + }, + /// `shl`, and `sshr`/`ushr`, by an immediate. + ShiftImm { + right: bool, + signed: bool, + size: u32, + q: bool, + shift: u32, + rd: u8, + rn: u8, + }, + /// `sshll`/`ushll` (`sxtl`/`uxtl` at shift 0); `upper` is the `2` form. + ShiftLong { + signed: bool, + size: u32, + shift: u32, + upper: bool, + rd: u8, + rn: u8, + }, + /// `shrn`/`shrn2`. + ShiftNarrow { + size: u32, + shift: u32, + upper: bool, + rd: u8, + rn: u8, + }, +} + +/// Decode `word` as one of these SIMD instructions, or `None` when it is not one. +pub fn decode(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = ((word >> 16) & 31) as u8; + let q = word & 0x4000_0000 != 0; + let u = word & 0x2000_0000 != 0; + let size = (word >> 22) & 3; + // Permute: bit 21 clear, bits 11:10 = 10, bits 14:12 pick the operation. + if word & 0xbf20_8c00 == 0x0e00_0800 { + let kind = match (word >> 12) & 7 { + 1 => Permute::Uzp1, + 2 => Permute::Trn1, + 3 => Permute::Zip1, + 5 => Permute::Uzp2, + 6 => Permute::Trn2, + 7 => Permute::Zip2, + _ => return None, + }; + // 64-bit lanes need the full vector. + if size == 3 && !q { + return None; + } + return Some(Op::Permute { + kind, + size, + q, + rd, + rn, + rm, + }); + } + // Two-register miscellaneous: bits 21:17 = 10000, bits 11:10 = 10. + if word & 0x9f3e_0c00 == 0x0e20_0800 { + let opcode = (word >> 12) & 0x1f; + let wide_needs_q = size != 3 || q; + return match (u, opcode) { + (false, 0) if size < 3 => Some(Op::Reverse { + group: 64, + size, + q, + rd, + rn, + }), + (true, 0) if size < 2 => Some(Op::Reverse { + group: 32, + size, + q, + rd, + rn, + }), + (false, 1) if size == 0 => Some(Op::Reverse { + group: 16, + size, + q, + rd, + rn, + }), + (false | true, 2) if size < 3 => Some(Op::PairLong { + signed: !u, + accumulate: false, + size, + q, + rd, + rn, + }), + (false | true, 6) if size < 3 => Some(Op::PairLong { + signed: !u, + accumulate: true, + size, + q, + rd, + rn, + }), + (false, 5) if size == 0 => Some(Op::Count { q, rd, rn }), + (true, 5) if size == 0 => Some(Op::Not { q, rd, rn }), + (true, 5) if size == 1 => Some(Op::ReverseBits { q, rd, rn }), + (false, 8) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Gt, + size, + q, + rd, + rn, + }), + (true, 8) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Ge, + size, + q, + rd, + rn, + }), + (true, 9) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Le, + size, + q, + rd, + rn, + }), + (false, 10) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Lt, + size, + q, + rd, + rn, + }), + (false, 11) if wide_needs_q => Some(Op::Abs { + negate: false, + size, + q, + rd, + rn, + }), + (true, 11) if wide_needs_q => Some(Op::Abs { + negate: true, + size, + q, + rd, + rn, + }), + (false, 0x12) if size < 3 => Some(Op::Narrow { + kind: Narrow::Plain, + size, + upper: q, + rd, + rn, + }), + (true, 0x12) if size < 3 => Some(Op::Narrow { + kind: Narrow::SignedToUnsignedSat, + size, + upper: q, + rd, + rn, + }), + (false, 0x14) if size < 3 => Some(Op::Narrow { + kind: Narrow::SignedSat, + size, + upper: q, + rd, + rn, + }), + (true, 0x14) if size < 3 => Some(Op::Narrow { + kind: Narrow::UnsignedSat, + size, + upper: q, + rd, + rn, + }), + // `cmeq #0` (U=0, opcode 9) is `SimdCompareZero`. + _ => None, + }; + } + // Three registers different: bit 21 set, bits 11:10 = 00. + if word & 0x9f20_0c00 == 0x0e20_0000 && size < 3 { + let kind = match (word >> 12) & 15 { + 0 => Long::Add, + 1 => Long::AddWide, + 2 => Long::Sub, + 3 => Long::SubWide, + 7 => Long::Abd, + 8 => Long::Mla, + 10 => Long::Mls, + 12 => Long::Mul, + _ => return None, + }; + return Some(Op::Long { + kind, + signed: !u, + size, + upper: q, + rd, + rn, + rm, + }); + } + // Shift by immediate: `immh` (bits 22:19) gives the element size and, with + // `immb`, the shift. `immh == 0` is the modified-immediate class instead. + if word & 0x9f80_0400 == 0x0f00_0400 && (word >> 19) & 15 != 0 { + let immh = (word >> 19) & 15; + let immhb = (word >> 16) & 0x7f; + // The position of the top set bit of immh: 0 for bytes up to 3 for doublewords. + let size = 31 - immh.leading_zeros(); + let esize = 8 << size; + return match (u, (word >> 11) & 0x1f) { + (false, 0b01010) if size < 3 || q => Some(Op::ShiftImm { + right: false, + signed: false, + size, + q, + shift: immhb - esize, + rd, + rn, + }), + (_, 0b00000) if size < 3 || q => Some(Op::ShiftImm { + right: true, + signed: !u, + size, + q, + shift: 2 * esize - immhb, + rd, + rn, + }), + (_, 0b10100) if size < 3 => Some(Op::ShiftLong { + signed: !u, + size, + shift: immhb - esize, + upper: q, + rd, + rn, + }), + (false, 0b10000) if size < 3 => Some(Op::ShiftNarrow { + size, + shift: 2 * esize - immhb, + upper: q, + rd, + rn, + }), + _ => None, + }; + } + // Scalar pairwise `addp Dd, Vn.2D`. + if word & 0xffff_fc00 == 0x5ef1_b800 { + return Some(Op::ScalarAddp { rd, rn }); + } + // Scalar `dup`: `mov Bd/Hd/Sd/Dd, Vn.T[index]`. + if word & 0xffe0_fc00 == 0x5e00_0400 { + let imm5 = (word >> 16) & 0x1f; + if imm5 == 0 || imm5.trailing_zeros() > 3 { + return None; + } + let size = imm5.trailing_zeros(); + return Some(Op::ScalarDup { + size, + index: imm5 >> (size + 1), + rd, + rn, + }); + } + // Scalar three-same, 64-bit only. + if word & 0xdf20_0400 == 0x5e20_0400 && size == 3 { + let kind = match (u, (word >> 11) & 0x1f) { + (false, 0x10) => Scalar::Add, + (true, 0x10) => Scalar::Sub, + (false, 0x06) => Scalar::Gt, + (true, 0x06) => Scalar::Hi, + (false, 0x07) => Scalar::Ge, + (true, 0x07) => Scalar::Hs, + (true, 0x11) => Scalar::Eq, + (false, 0x11) => Scalar::Tst, + _ => return None, + }; + return Some(Op::ScalarThree { kind, rd, rn, rm }); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classification_declines_what_it_does_not_implement() { + // cmeq #0 is `SimdCompareZero`; half-width lanes need Q for 64-bit lanes; + // `ins`, `umov` and the three-same group belong to other decoders. + assert_eq!(decode(0x0e209820), None); + assert_eq!(decode(0x0ee0b820), None, "abs .1d is reserved"); + assert_eq!(decode(0x2ee21c20), None, "bif is three-same"); + assert_eq!(decode(0x4e083c20), None, "umov"); + assert!(decode(0x4e821820).is_some()); + } +} diff --git a/src/target/src/aarch64/simd_struct.rs b/src/isa/aarch64/src/simd_struct.rs similarity index 100% rename from src/target/src/aarch64/simd_struct.rs rename to src/isa/aarch64/src/simd_struct.rs diff --git a/src/lift/aarch64/Cargo.toml b/src/lift/aarch64/Cargo.toml index 90e4cf8..865dbe8 100644 --- a/src/lift/aarch64/Cargo.toml +++ b/src/lift/aarch64/Cargo.toml @@ -9,6 +9,9 @@ path = "src/lib.rs" [dependencies] volt-ir.workspace = true +volt-isa-aarch64.workspace = true + +[dev-dependencies] volt-target = { path = "../../target", default-features = false, features = ["aarch64"] } [lints] diff --git a/src/lift/aarch64/src/builder.rs b/src/lift/aarch64/src/builder.rs index 466b07d..41df9ee 100644 --- a/src/lift/aarch64/src/builder.rs +++ b/src/lift/aarch64/src/builder.rs @@ -4,7 +4,7 @@ use volt_ir::{ function::{self as ir, BinOp as B, CmpOp as C, Function, Opcode, Value}, types::{IntDesc, Type, TypeKind}, }; -use volt_target::aarch64::decode::{Addressing, Width}; +use volt_isa_aarch64::decode::{Addressing, Width}; /// Emits IR for one lifted function and knows where guest state lives. /// @@ -251,7 +251,10 @@ impl Builder { pub fn address(&self, rn: u8, a: Addressing) -> (Value, Option) { let base = self.reg(self.i64, rn, false); match a { - Addressing::Offset(n) => (self.imm(self.i64, B::Add, base, n as u64), None), + // The machines do not tell the unprivileged `ldtr`/`sttr` from a plain access. + Addressing::Offset(n) | Addressing::Unscaled(n) | Addressing::Unprivileged(n) => { + (self.imm(self.i64, B::Add, base, n as u64), None) + } Addressing::PreIndex(n) => { let address = self.imm(self.i64, B::Add, base, n as u64); let writeback = Writeback { @@ -275,7 +278,7 @@ impl Builder { self.i64, B::Add, base, - self.imm(self.i64, B::Shl, v, amount as u64), + self.imm(self.i64, B::Shl, v, u64::from(amount.unwrap_or(0))), ); (address, None) } diff --git a/src/lift/aarch64/src/int.rs b/src/lift/aarch64/src/int.rs index 333ffe0..f23cd07 100644 --- a/src/lift/aarch64/src/int.rs +++ b/src/lift/aarch64/src/int.rs @@ -5,7 +5,7 @@ use volt_ir::{ function::{BinOp as B, CmpOp as C, Value}, types::Type, }; -use volt_target::aarch64::decode::*; +use volt_isa_aarch64::decode::*; impl Builder { pub(crate) fn not(&self, v: Value) -> Value { diff --git a/src/lift/aarch64/src/lib.rs b/src/lift/aarch64/src/lib.rs index 000962e..dcbdfa7 100644 --- a/src/lift/aarch64/src/lib.rs +++ b/src/lift/aarch64/src/lib.rs @@ -34,13 +34,15 @@ //! families) is architecture and is lifted here. //! //! The lifter lifts instructions; the *caller* owns the block: it decodes, -//! decides where a block ends (see [`Instruction::terminates_with`]), and -//! calls [`Lifter::exit`] when a block runs off its end without a -//! terminating instruction. An instruction that falls through leaves the -//! [`Builder`] positioned for the next one. An instruction that does not (a -//! branch, or a trap the environment ends with [`Environment::exit`]) has -//! already terminated the current block. An environment must therefore end the -//! block for every forwarded instruction for which `terminates_with` is true. +//! decides where a block ends, and calls [`Lifter::exit`] when a block runs +//! off its end without a terminating instruction. Architectural branches come +//! from `Instruction::flow`; any further cut (an emulator ending a block at +//! every access it hands to the machine, say) is the caller's policy. An +//! instruction that falls through leaves the [`Builder`] positioned for the +//! next one. An instruction that does not (a branch, or a trap the environment +//! ends with [`Environment::exit`]) has already terminated the current block. +//! An environment must therefore end the block for every forwarded instruction +//! that the caller's policy treats as ending it. #![no_std] extern crate alloc; @@ -56,7 +58,7 @@ mod tests; pub use builder::Builder; use core::fmt; use volt_ir::function::{BinOp as B, CmpOp as C, Function, Value}; -use volt_target::aarch64::decode::*; +use volt_isa_aarch64::decode::*; /// A load, store or other guest memory instruction handed to /// [`Environment::memory`]. @@ -206,7 +208,7 @@ impl Lifter { Instruction::ArithImm(a) => { let t = b.ty(a.width); let lhs = b.reg(t, a.rn, false); - let rhs = b.k(t, a.immediate as u64); + let rhs = b.k(t, u64::from(a.operand())); let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); if a.flags { b.put(a.rd, r); @@ -414,7 +416,7 @@ impl Lifter { } Instruction::Indirect(a) => { let target = b.reg(b.i64, a.rn, false); - if a.link { + if a.kind == IndirectKind::Call { b.put(30, b.k(b.i64, pc.wrapping_add(4))) } env.exit(b, target); @@ -448,7 +450,6 @@ impl Lifter { memory(env, b, pc, &insn, None, None)? } Instruction::SimdImm(a) => simd::imm(b, a), - Instruction::SimdMove { rd, rn } => simd::mov(b, rd, rn), Instruction::SimdInsert(a) => simd::insert(b, a), Instruction::SimdInsertElement(a) => simd::insert_element(b, a), Instruction::SimdExtract(a) => simd::extract(b, a), @@ -464,6 +465,7 @@ impl Lifter { | Instruction::Hint(_) | Instruction::Prefetch(_) | Instruction::PrefetchLiteral(_) + | Instruction::RangePrefetch(_) | Instruction::Dsb(_) | Instruction::Dmb(_) | Instruction::Yield => {} @@ -482,6 +484,14 @@ impl Lifter { | Instruction::Clrex(_) | Instruction::Psci | Instruction::CacheOp(_) => env.event(b, pc, &insn)?, + // Decoded by the same decoder, but the lifter has no semantics for these + // families: a machine that wants them runs them itself. + Instruction::Fp(_) + | Instruction::Simd(_) + | Instruction::Pauth(_) + | Instruction::Gxf(_) => { + return Err(LiftError::Unsupported); + } } Ok(()) } diff --git a/src/lift/aarch64/src/simd.rs b/src/lift/aarch64/src/simd.rs index a9d24c7..5b9e0ec 100644 --- a/src/lift/aarch64/src/simd.rs +++ b/src/lift/aarch64/src/simd.rs @@ -6,11 +6,11 @@ use crate::Builder; use alloc::vec::Vec; use volt_ir::function::{BinOp as B, CmpOp as C, Value}; -use volt_target::aarch64::decode::*; +use volt_isa_aarch64::decode::*; pub(crate) fn imm(b: &Builder, a: SimdImmediate) { let at = b.v_at(a.rd); match a.combine { - ImmCombine::Set => { + ImmCombine::Set | ImmCombine::SetInverted => { b.store(at, b.k(b.i64, a.low)); b.store(at + 8, b.k(b.i64, a.high)); } @@ -32,14 +32,6 @@ pub(crate) fn imm(b: &Builder, a: SimdImmediate) { } } } -pub(crate) fn mov(b: &Builder, rd: u8, rn: u8) { - let at_d = b.v_at(rd); - let at_n = b.v_at(rn); - let lo = b.load(b.i64, at_n); - let hi = b.load(b.i64, at_n + 8); - b.store(at_d, lo); - b.store(at_d + 8, hi); -} pub(crate) fn insert(b: &Builder, a: SimdInsert) { // Replace one lane of the destination with the low bits of a GPR. let ebits = (1u64 << a.esize) * 8; @@ -185,7 +177,21 @@ pub(crate) fn ext(b: &Builder, a: SimdExt) { let vn_hi = b.load(b.i64, at_n + 8); let vm_lo = b.load(b.i64, at_m); let vm_hi = b.load(b.i64, at_m + 8); - let (lo, hi) = if a.imm == 0 { + let (lo, hi) = if !a.q { + // `.8b`: a byte window over the low halves of `rm:rn`; the upper half clears. + let lo = if a.imm == 0 { + vn_lo + } else { + let s = a.imm as u64 * 8; + b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vn_lo, s), + b.imm(b.i64, B::Shl, vm_lo, 64 - s), + ) + }; + (lo, b.k(b.i64, 0)) + } else if a.imm == 0 { (vn_lo, vn_hi) } else if a.imm == 8 { (vn_hi, vm_lo) @@ -341,6 +347,8 @@ pub(crate) fn table(b: &Builder, a: SimdTable) { } pub(crate) fn fmov(b: &Builder, a: SimdFmov) { // Pure bit move; no FP arithmetic involved. + // `fmov Vd.D[1], Xn` writes only the upper doubleword; `fmov Xd, Vn.D[1]` reads it. + let half = if a.upper { 8 } else { 0 }; if a.to_fp { let v = b.reg(b.i64, a.rn, true); let bits = if a.double { @@ -349,11 +357,13 @@ pub(crate) fn fmov(b: &Builder, a: SimdFmov) { b.imm(b.i64, B::BitAnd, v, 0xffff_ffff) }; let at = b.v_at(a.rd); - b.store(at, bits); - b.store(at + 8, b.k(b.i64, 0)); + b.store(at + half, bits); + if !a.upper { + b.store(at + 8, b.k(b.i64, 0)); + } } else { let at = b.v_at(a.rn); - let v = b.load(b.i64, at); + let v = b.load(b.i64, at + half); let bits = if a.double { v } else { diff --git a/src/lift/aarch64/src/tests.rs b/src/lift/aarch64/src/tests.rs index be95711..2f88f2a 100644 --- a/src/lift/aarch64/src/tests.rs +++ b/src/lift/aarch64/src/tests.rs @@ -4,10 +4,8 @@ use crate::{Builder, Environment, LiftError, Lifter, MemoryAccess}; use alloc::vec::Vec; use core::mem::offset_of; use volt_ir::function::{BinOp as B, Function, Value}; -use volt_target::{ - aarch64::decode::*, - native::{self, JittedFunction}, -}; +use volt_isa_aarch64::{decode::*, flow::Flow}; +use volt_target::native::{self, JittedFunction}; /// Guest state of the test environment. #[repr(C)] @@ -104,7 +102,7 @@ impl Environment for Flat { Self::load(b, address, 0, a.size, a.rt, a.signed) } (Instruction::Literal(a), Some(address)) => { - Self::load(b, address, 0, a.size, a.rt, SignExtend::None) + Self::load(b, address, 0, a.size, a.rt, a.signed) } (Instruction::Pair(a), Some(address)) if a.op == MemoryOp::Store => { Self::store(b, address, 0, a.size, a.rt); @@ -136,8 +134,8 @@ impl Environment for Flat { } } -/// Lift `bytes` as one block at `pc`: instructions until one terminates it -/// (plain memory accesses do not), then an exit at the next pc. +/// Lift `bytes` as one block at `pc`: instructions until an architectural branch +/// ends it (plain memory accesses do not), then an exit at the next pc. fn lift(pc: u64, bytes: &[u8]) -> Result<(Function, Flat), LiftError> { let mut lifter = Lifter::new(Flat::default()); let mut at = pc; @@ -145,7 +143,7 @@ fn lift(pc: u64, bytes: &[u8]) -> Result<(Function, Flat), LiftError> { for chunk in bytes.chunks_exact(4) { let word = u32::from_le_bytes(chunk.try_into().unwrap()); let instruction = decode(word).unwrap(); - ended = instruction.terminates_with(true); + ended = instruction.flow(at) != Flow::Next; lifter.lift(at, instruction)?; at += 4; if ended { @@ -459,6 +457,65 @@ fn simd_dup_from_element_replicates_a_lane() { } } #[test] +fn simd_ext_uses_the_arrangement_it_encodes() { + let setup = || { + let mut cpu = State::default(); + cpu.v[0] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + cpu.v[2] = [0x1112_1314_1516_1718, 0x191a_1b1c_1d1e_1f20]; + cpu + }; + // ext v0.8b, v1.8b, v2.8b, #3 windows the low halves and clears the upper half. + let mut cpu = setup(); + compile(0x1000, &bytes(&[0x2e021820])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.v[0], [0x1617_1801_0203_0405, 0]); + // ext v0.16b, v1.16b, v2.16b, #3. + let mut cpu = setup(); + compile(0x1000, &bytes(&[0x6e021820])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.v[0], [0x0e0f_1001_0203_0405, 0x1617_1809_0a0b_0c0d]); +} +#[test] +fn simd_modified_immediates_write_the_expanded_pattern() { + for (word, want) in [ + // movi v0.2d, #0 and movi d0, #0xff (which clears the upper half). + (0x6f00e400u32, [0u64, 0]), + (0x2f00e420, [0xff, 0]), + // mvni v31.4s, #0xff inverts every 32-bit lane. + (0x6f0707ff, [0xffff_ff00_ffff_ff00; 2]), + // fmov v0.8h/.4h, #2.0 and #-2.5 expand the half-precision pattern. + (0x4f00fc00, [0x4000_4000_4000_4000; 2]), + (0x0f00fc00, [0x4000_4000_4000_4000, 0]), + (0x4f04fc80, [0xc100_c100_c100_c100; 2]), + ] { + let rd = (word & 31) as usize; + let mut cpu = State::default(); + cpu.v[rd] = [0x5555_5555_5555_5555; 2]; + compile(0x1000, &bytes(&[word])).unwrap().run(&mut cpu); + assert_eq!(cpu.v[rd], want, "{word:08x}"); + } +} +#[test] +fn simd_fmov_upper_doubleword_keeps_the_lower_half() { + let mut cpu = State::default(); + cpu.v[0] = [0x1111_1111_1111_1111, 0x2222_2222_2222_2222]; + cpu.x[3] = 0x3333_3333_3333_3333; + // fmov v0.d[1], x3. + compile(0x1000, &bytes(&[0x9eaf0060])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.v[0], [0x1111_1111_1111_1111, 0x3333_3333_3333_3333]); + // fmov x4, v0.d[1]. + cpu.v[0][1] = 0x4444_4444_4444_4444; + compile(0x1000, &bytes(&[0x9eae0004])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.x[4], 0x4444_4444_4444_4444); +} +#[test] fn simd_table_lookup_selects_bytes() { // tbl v0.16b, {v1.16b, v2.16b}, v3.16b with indices 0,2,4,... let block = compile(0x1000, &bytes(&[0x4e032020])).unwrap(); diff --git a/src/target/Cargo.toml b/src/target/Cargo.toml index e41cbec..d50e7bf 100644 --- a/src/target/Cargo.toml +++ b/src/target/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [dependencies] object.workspace = true volt-ir.workspace = true +volt-isa-aarch64.workspace = true libc.workspace = true [lints] diff --git a/src/target/src/aarch64/disasm.rs b/src/target/src/aarch64/disasm.rs deleted file mode 100644 index a484f6e..0000000 --- a/src/target/src/aarch64/disasm.rs +++ /dev/null @@ -1,1611 +0,0 @@ -//! AArch64 (A64) disassembler: the inverse of [`super::encode`]. -//! -//! It decodes exactly the instructions the encoder emits into a readable -//! listing, so codegen debugging does not mean hand-decoding hex. Field -//! positions mirror the encoder bit-for-bit. Anything outside the emitted -//! subset prints as `.word 0x` so nothing is silently mis-decoded. -//! -//! [`one`] renders a single 32-bit instruction word; [`format`] renders a -//! whole code buffer with byte addresses. -//! -//! This module is intentionally self-contained (it operates on raw words and -//! shares no types with the encoder) so it keeps compiling while sibling -//! modules evolve. - -use alloc::string::{String, ToString as _}; -use core::fmt::Write as _; - -/// A source-line-table row: the byte offset where a source line's code begins. -pub struct SourceLine { - pub offset: u32, - pub line: u32, -} - -/// A function's location in a linked image: its name and its word offset. -pub struct Sym<'a> { - pub name: &'a str, - pub word: usize, -} - -/// A defined symbol by absolute address (a function or data object), for -/// resolving branch and `adrp` targets when disassembling an ELF image. -pub struct AddrSym<'a> { - pub name: &'a str, - pub addr: u64, - pub size: u64, - pub is_func: bool, -} - -/// A decoded source-line row keyed by absolute address (from a `.debug_line` -/// program), for objdump `-S`-style annotation of a real ELF. -pub struct AddrLine { - pub addr: u64, - pub line: u32, -} - -/// The context that makes a listing absolute: the current instruction's -/// address and the image's symbols. When present, PC-relative targets render -/// as `0x ` (objdump style). -#[derive(Clone, Copy)] -struct Context<'a> { - pc: u64, - syms: &'a [AddrSym<'a>], -} - -/// Render one instruction word as `mnemonic operands`. -pub fn one(word: u32) -> String { - let mut out = String::new(); - decode(&mut out, word, None); - out -} - -/// Render a whole instruction stream (one word per line, `addr: hex text`). -pub fn format(code: &[u32]) -> String { - let mut out = String::new(); - for (i, &word) in code.iter().enumerate() { - let _ = write!(out, "{:04x}: {:08x} ", i * 4, word); - decode(&mut out, word, None); - out.push('\n'); - } - out -} - -/// Render a listing with source-line markers interleaved (objdump `-S` -/// style): a `; line N` header precedes the instructions belonging to that -/// source line. `lines` must be sorted by offset (as isel produces). -pub fn format_with_lines(code: &[u32], lines: &[SourceLine]) -> String { - let mut out = String::new(); - let mut li: usize = 0; - for (i, &word) in code.iter().enumerate() { - let byte_off: u32 = (i * 4) as u32; - while li < lines.len() && lines[li].offset == byte_off { - let _ = writeln!(out, "; line {}", lines[li].line); - li += 1; - } - let _ = write!(out, "{byte_off:04x}: {word:08x} "); - decode(&mut out, word, None); - out.push('\n'); - } - out -} - -/// Render a whole linked module: each function gets a `name:` label at its -/// offset, and every resolved `bl` is annotated with the callee's name -/// (``), so a linked image reads like a symbolized listing. -pub fn format_module(code: &[u32], syms: &[Sym<'_>]) -> String { - let mut out = String::new(); - for (i, &word) in code.iter().enumerate() { - for s in syms { - if s.word == i { - if i != 0 { - out.push('\n'); - } - let _ = writeln!(out, "{}:", s.name); - } - } - let _ = write!(out, "{:04x}: {:08x} ", i * 4, word); - decode(&mut out, word, None); - // Annotate any PC-relative branch (call or local) with the function - // it lands in, as `` or ``, the way objdump does. - if let Some(target) = branch_target_word(word, i) - && let Some(s) = containing_sym(syms, target) - { - let off = target - s.word as i64; - if off == 0 { - let _ = write!(out, " <{}>", s.name); - } else { - let _ = write!(out, " <{}+0x{:x}>", s.name, off as u64 * 4); - } - } - out.push('\n'); - } - out -} - -/// Render an ELF code section as an objdump-style listing: absolute -/// addresses, function labels, and branch/`adrp` targets resolved to -/// `0x `. `base` is the section's load address; `syms` are all -/// defined symbols (functions and data). -pub fn format_elf(code: &[u32], base: u64, syms: &[AddrSym<'_>]) -> String { - format_elf_impl(code, base, syms, &[]) -} - -/// Like [`format_elf`], but interleaves `; line N` markers from `lines` -/// (sorted by address) before the instruction at each address, so a listing -/// of a `-g` object reads with its source lines. -pub fn format_elf_with_lines( - code: &[u32], - base: u64, - syms: &[AddrSym<'_>], - lines: &[AddrLine], -) -> String { - format_elf_impl(code, base, syms, lines) -} - -fn format_elf_impl(code: &[u32], base: u64, syms: &[AddrSym<'_>], lines: &[AddrLine]) -> String { - let mut out = String::new(); - let mut li: usize = 0; - for (i, &word) in code.iter().enumerate() { - let addr = base + i as u64 * 4; - for s in syms { - if s.is_func && s.addr == addr { - if i != 0 { - out.push('\n'); - } - let _ = writeln!(out, "{addr:016x} <{}>:", s.name); - } - } - while li < lines.len() && lines[li].addr < addr { - li += 1; // skip any before this insn - } - while li < lines.len() && lines[li].addr == addr { - let _ = writeln!(out, "; line {}", lines[li].line); - li += 1; - } - let _ = write!(out, "{addr:08x}: {word:08x} "); - decode(&mut out, word, Some(Context { pc: addr, syms })); - out.push('\n'); - } - out -} - -/// Render a PC-relative code target `rel_bytes` from the current PC: -/// absolute `0x ` when a context is present, else the -/// self-contained `.±N` form. -fn render_target(out: &mut String, ctx: Option>, rel_bytes: i64) { - if let Some(c) = ctx { - // Wrapping add (like the adrp path): a disassembler must never panic - // on the bytes it is given, including a branch whose displacement - // runs below zero. - let abs: u64 = (c.pc as i64).wrapping_add(rel_bytes) as u64; - let _ = write!(out, "0x{abs:x}"); - annotate_sym(out, c.syms, abs); - } else { - out.push('.'); - out.push(if rel_bytes < 0 { '-' } else { '+' }); - out.push_str(&rel_bytes.unsigned_abs().to_string()); - } -} - -/// Append ` ` for the symbol that contains `addr`, if any. -fn annotate_sym(out: &mut String, syms: &[AddrSym<'_>], addr: u64) { - if let Some((name, off)) = symbol_at(syms, addr) { - if off == 0 { - let _ = write!(out, " <{name}>"); - } else { - let _ = write!(out, " <{name}+0x{off:x}>"); - } - } -} - -/// The symbol containing `addr` (the greatest-addressed symbol at or before -/// it): its name and the byte offset into it. -fn symbol_at<'a>(syms: &'a [AddrSym<'a>], addr: u64) -> Option<(&'a str, u64)> { - let mut best: Option<&AddrSym<'_>> = None; - for s in syms { - if s.addr <= addr && best.is_none_or(|b| s.addr > b.addr) { - best = Some(s); - } - } - best.map(|s| (s.name, addr - s.addr)) -} - -/// The target word index of a PC-relative branch (b/bl/b.cond/cbz/cbnz/ -/// tbz/tbnz) at `index`, or `None` if `w` is not such a branch. -fn branch_target_word(w: u32, index: usize) -> Option { - let base: i64 = index as i64; - if w & 0x7C00_0000 == 0x1400_0000 { - // b / bl: imm26 - return Some(base + (((w << 6) as i32) >> 6) as i64); - } - if w & 0xFF00_0010 == 0x5400_0000 || w & 0x7E00_0000 == 0x3400_0000 { - // b.cond / cbz / cbnz: imm19 - return Some(base + (((w << 8) as i32) >> 13) as i64); - } - if w & 0x7E00_0000 == 0x3600_0000 { - // tbz / tbnz: imm14 - return Some(base + (((w << 13) as i32) >> 18) as i64); - } - None -} - -/// The symbol whose range contains word `target` (the greatest symbol at or -/// before it), or `None`. -fn containing_sym<'a>(syms: &'a [Sym<'a>], target: i64) -> Option<&'a Sym<'a>> { - if target < 0 { - return None; - } - let mut best: Option<&'a Sym<'a>> = None; - for s in syms { - if s.word as i64 <= target && best.is_none_or(|b| s.word > b.word) { - best = Some(s); - } - } - best -} - -fn rd(w: u32) -> u32 { - w & 0x1F -} -fn rn(w: u32) -> u32 { - (w >> 5) & 0x1F -} -fn rm(w: u32) -> u32 { - (w >> 16) & 0x1F -} -fn ra(w: u32) -> u32 { - (w >> 10) & 0x1F -} -fn sf(w: u32) -> bool { - (w >> 31) & 1 != 0 -} - -/// A 32/64-bit general register (`w`/`x`), rendering 31 as the zero register. -fn gp(out: &mut String, x64: bool, r: u32) { - let c: char = if x64 { 'x' } else { 'w' }; - if r == 31 { - let _ = write!(out, "{c}zr"); - } else { - let _ = write!(out, "{c}{r}"); - } -} - -/// A general register in an address/stack context, rendering 31 as `sp`. -fn sp(out: &mut String, x64: bool, r: u32) { - if r == 31 { - out.push_str("sp"); - } else { - gp(out, x64, r); - } -} - -fn cond_name(v: u32) -> &'static str { - // binutils spelling (cs/cc rather than the hs/lo synonyms) so listings - // match objdump. - match v { - 0 => "eq", - 1 => "ne", - 2 => "cs", - 3 => "cc", - 4 => "mi", - 5 => "pl", - 6 => "vs", - 7 => "vc", - 8 => "hi", - 9 => "ls", - 10 => "ge", - 11 => "lt", - 12 => "gt", - 13 => "le", - 14 => "al", - 15 => "nv", - _ => "??", - } -} - -/// Masks for the three-register data-processing form: clear rd, rn, rm; keep -/// the opcode and the op2/shift field in bits[10:15]. The `sf` bit is dropped -/// so one entry covers both the w and x forms. -const RRR_MASK: u32 = 0x7FE0_FC00; - -/// `RRR_MASK` with the imm6 shift-amount field (bits[15:10]) dropped, for the -/// add/sub shifted-register form where that field is a real operand and not -/// part of the opcode. -const RRR_NO_IMM6_MASK: u32 = RRR_MASK & !0xFC00u32; - -/// The 32-bit-base opcodes of the shared three-register form (sf dropped). -const RRR_TABLE: &[(u32, &str)] = &[ - (0x0B00_0000, "add"), - (0x4B00_0000, "sub"), - (0x0A00_0000, "and"), - (0x6A00_0000, "ands"), // flag-setting and (shift 0) - (0x2A00_0000, "orr"), - (0x4A00_0000, "eor"), - (0x1AC0_0C00, "sdiv"), - (0x1AC0_0800, "udiv"), - (0x1AC0_2000, "lsl"), - (0x1AC0_2400, "lsr"), - (0x1AC0_2800, "asr"), -]; - -/// `ret` (`ret x30`): the exact operandless encoding, matched first. -const RET_WORD: u32 = 0xD65F_0000 | (30 << 5); - -fn decode(out: &mut String, w: u32, ctx: Option>) { - // Exact, operandless / special encodings first. - if w == RET_WORD { - out.push_str("ret"); - return; - } - if w & 0xFFE0_001F == 0xD400_0001 { - let _ = write!(out, "svc #{}", (w >> 5) & 0xFFFF); - return; - } - if w & 0xFFFF_FC1F == 0xD63F_0000 { - out.push_str("blr "); - sp(out, true, rn(w)); - return; - } - - // `mov` (register) is `orr Xd, xzr, Xm` with rn == 31; render it as a move. - if w & 0x7FE0_FFE0 == 0x2A00_03E0 { - out.push_str("mov "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - - // neg Rd, Rm = sub Rd, RZR, Rm (rn == 31), the objdump alias. - if w & 0x7FE0_FC00 == 0x4B00_0000 && rn(w) == 31 { - out.push_str("neg "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - - // The shared three-register form. - for &(m, mnem) in RRR_TABLE { - if w & RRR_MASK == m { - out.push_str(mnem); - out.push(' '); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - } - - // add/sub, shifted-register form with a nonzero LSL shift amount (imm6 at - // bits[15:10]). The shift-0 case is already handled by the table above. - if w & RRR_NO_IMM6_MASK == 0x0B00_0000 || w & RRR_NO_IMM6_MASK == 0x4B00_0000 { - let is_sub = w & RRR_NO_IMM6_MASK == 0x4B00_0000; - out.push_str(if is_sub { "sub " } else { "add " }); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - let _ = write!(out, ", lsl #{}", (w >> 10) & 0x3F); - return; - } - - // madd/msub (mul is madd with ra == 31). - if w & 0x7FE0_8000 == 0x1B00_0000 || w & 0x7FE0_8000 == 0x1B00_8000 { - let is_sub = (w >> 15) & 1 != 0; - if !is_sub && ra(w) == 31 { - out.push_str("mul "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - if is_sub && ra(w) == 31 { - // mneg Rd, Rn, Rm = msub Rd, Rn, Rm, RZR - out.push_str("mneg "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - out.push_str(if is_sub { "msub " } else { "madd " }); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - out.push_str(", "); - gp(out, sf(w), ra(w)); - return; - } - - // add/sub immediate (bit30 = op is sub, bit29 = S sets flags, bit22 = - // shift-by-12). When S is set and rd == 31 the alias is cmp / cmn. Rn/Rd - // 31 otherwise mean sp. - if w & 0x1F80_0000 == 0x1100_0000 { - let is_sub = (w >> 30) & 1 != 0; - let setflags = (w >> 29) & 1 != 0; - let shift12 = (w >> 22) & 1 != 0; - let imm: u32 = (w >> 10) & 0xFFF; - if setflags && rd(w) == 31 { - out.push_str(if is_sub { "cmp " } else { "cmn " }); - sp(out, sf(w), rn(w)); - let _ = write!(out, ", #{imm}"); - if shift12 { - out.push_str(", lsl #12"); - } - return; - } - // mov Rd, Rn (to/from SP) = add Rd, Rn, #0 with no shift. - if !setflags && !is_sub && imm == 0 && !shift12 && (rd(w) == 31 || rn(w) == 31) { - out.push_str("mov "); - sp(out, sf(w), rd(w)); - out.push_str(", "); - sp(out, sf(w), rn(w)); - return; - } - let mnem = if setflags { - if is_sub { "subs" } else { "adds" } - } else if is_sub { - "sub" - } else { - "add" - }; - out.push_str(mnem); - out.push(' '); - sp(out, sf(w), rd(w)); - out.push_str(", "); - sp(out, sf(w), rn(w)); - let _ = write!(out, ", #{imm}"); - if shift12 { - out.push_str(", lsl #12"); - } - return; - } - - // movn (move wide negated): rendered as `mov Rd, #<~imm>`, matching - // objdump's alias. - if w & 0x7F80_0000 == 0x1280_0000 { - let imm: u64 = ((w >> 5) & 0xFFFF) as u64; - let shift: u32 = ((w >> 21) & 3) * 16; - let val = !(imm << shift); - out.push_str("mov "); - gp(out, sf(w), rd(w)); - if sf(w) { - let _ = write!(out, ", #0x{val:x}"); - } else { - let _ = write!(out, ", #0x{:x}", val & 0xFFFF_FFFF); - } - return; - } - - // movz / movk (32/64-bit, with an optional lsl of the 16-bit immediate). - // objdump aliases movz to `mov Rd, #>`, so render the - // assembled value for movz. - if w & 0x7F80_0000 == 0x5280_0000 || w & 0x7F80_0000 == 0x7280_0000 { - let keep = (w >> 29) & 1 != 0; // opc 11 -> movk, 10 -> movz - let imm: u64 = ((w >> 5) & 0xFFFF) as u64; - let shift: u32 = ((w >> 21) & 3) * 16; - if !keep { - // movz -> mov Rd, #value - let val = imm << shift; - out.push_str("mov "); - gp(out, sf(w), rd(w)); - if sf(w) { - let _ = write!(out, ", #0x{val:x}"); - } else { - let _ = write!(out, ", #0x{:x}", val & 0xFFFF_FFFF); - } - return; - } - out.push_str("movk "); - gp(out, sf(w), rd(w)); - let _ = write!(out, ", #{imm}"); - if shift != 0 { - let _ = write!(out, ", lsl #{shift}"); - } - return; - } - - // adds/subs (shifted register, shift 0), with cmn/cmp aliases when - // rd == 31. - if w & 0x7FE0_FC00 == 0x2B00_0000 || w & 0x7FE0_FC00 == 0x6B00_0000 { - let is_sub = (w >> 30) & 1 != 0; - if rd(w) == 31 { - out.push_str(if is_sub { "cmp " } else { "cmn " }); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - if is_sub && rn(w) == 31 { - // negs Rd, Rm = subs Rd, RZR, Rm - out.push_str("negs "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - out.push_str(if is_sub { "subs " } else { "adds " }); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - - // cset (csinc Rd, zr, zr, invert(cond)). The mask excludes the cond field - // [12:15]. - if w & 0x7FFF_0FE0 == 0x1A9F_07E0 { - out.push_str("cset "); - gp(out, sf(w), rd(w)); - let _ = write!(out, ", {}", cond_name(((w >> 12) & 0xF) ^ 1)); - return; - } - - // csel Rd, Rn, Rm, cond. - if w & 0x7FE0_0C00 == 0x1A80_0000 { - out.push_str("csel "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - let _ = write!(out, ", {}", cond_name((w >> 12) & 0xF)); - return; - } - - // cbz / cbnz Rt, label (bit24 selects nz). - if w & 0x7E00_0000 == 0x3400_0000 { - let nz = (w >> 24) & 1 != 0; - let imm19: i32 = ((w << 8) as i32) >> 13; // sign-extend bits[5:23] - out.push_str(if nz { "cbnz " } else { "cbz " }); - gp(out, sf(w), rd(w)); - out.push_str(", "); - render_target(out, ctx, imm19 as i64 * 4); - return; - } - - // tbz / tbnz Rt, #bit, label (bit24 selects nz). Test-bit index is b5:b40. - if w & 0x7E00_0000 == 0x3600_0000 { - let nz = (w >> 24) & 1 != 0; - let bit: u32 = (((w >> 31) & 1) << 5) | ((w >> 19) & 0x1F); - let imm14: i32 = (((w << 13) as i32) >> 18) * 4; // bits[18:5], scaled - out.push_str(if nz { "tbnz " } else { "tbz " }); - gp(out, (w >> 31) & 1 != 0, rd(w)); - let _ = write!(out, ", #{bit}, "); - render_target(out, ctx, imm14 as i64); - return; - } - - // b.cond label (bit4 == 0 distinguishes it from the - // consistent-conditional-branch forms). - if w & 0xFF00_0010 == 0x5400_0000 { - let imm19: i32 = (((w << 8) as i32) >> 13) * 4; - let _ = write!(out, "b.{} ", cond_name(w & 0xF)); - render_target(out, ctx, imm19 as i64); - return; - } - - // b / bl label (imm26 << 2). - if w & 0xFC00_0000 == 0x1400_0000 || w & 0xFC00_0000 == 0x9400_0000 { - let link = (w >> 31) & 1 != 0; - let imm26: i32 = (((w << 6) as i32) >> 6) << 2; - out.push_str(if link { "bl " } else { "b " }); - render_target(out, ctx, imm26 as i64); - return; - } - - // stp/ldp pair, pre/post-index (64-bit). - if w & 0xFFC0_0000 == 0xA980_0000 || w & 0xFFC0_0000 == 0xA8C0_0000 { - let load = (w >> 22) & 1 != 0; - let imm7: i32 = (((w << 10) as i32) >> 25) * 8; // bits[15:21], scaled by 8 - out.push_str(if load { "ldp " } else { "stp " }); - gp(out, true, rd(w)); - out.push_str(", "); - gp(out, true, ra(w)); // rt2 is bits[10:15] - out.push_str(", ["); - sp(out, true, rn(w)); - if load { - // post-index: `[base], #imm` - let _ = write!(out, "], #{imm7}"); - } else { - // pre-index: `[base, #imm]!` - let _ = write!(out, ", #{imm7}]!"); - } - return; - } - - // Logical immediate: and/orr/eor/ands, with mov (orr from zr) and tst - // (ands to zr) aliases. - if w & 0x1F80_0000 == 0x1200_0000 { - let opc = (w >> 29) & 3; - let nn: u32 = (w >> 22) & 1; - let immr: u32 = (w >> 16) & 0x3F; - let imms: u32 = (w >> 10) & 0x3F; - if let Some(val) = decode_bitmask(sf(w), nn, imms, immr) { - if opc == 1 && rn(w) == 31 { - // mov Rd, #imm - out.push_str("mov "); - gp(out, sf(w), rd(w)); - let _ = write!(out, ", #0x{val:x}"); - return; - } - if opc == 3 && rd(w) == 31 { - // tst Rn, #imm - out.push_str("tst "); - gp(out, sf(w), rn(w)); - let _ = write!(out, ", #0x{val:x}"); - return; - } - let mnem = match opc { - 0 => "and", - 1 => "orr", - 2 => "eor", - _ => "ands", - }; - out.push_str(mnem); - out.push(' '); - sp(out, sf(w), rd(w)); // and/orr/eor immediate can target sp - out.push_str(", "); - gp(out, sf(w), rn(w)); - let _ = write!(out, ", #0x{val:x}"); - return; - } - } - - // extr Rd, Rn, Rm, #lsb (ror when rn == rm). bit23 == 1 separates it from - // the bitfield ops. - if w & 0x7F80_0000 == 0x1380_0000 { - let lsb: u32 = (w >> 10) & 0x3F; - if rn(w) == rm(w) { - out.push_str("ror "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - let _ = write!(out, ", #{lsb}"); - } else { - out.push_str("extr "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - let _ = write!(out, ", #{lsb}"); - } - return; - } - - // STUR/LDUR: unscaled signed 9-bit offset (integer). size 00=b 01=h 10=w - // 11=x; opc picks store / load / sign-extending load. - if w & 0x3F20_0C00 == 0x3800_0000 { - let size = w >> 30; - let opc = (w >> 22) & 3; - let imm9: i32 = ((w << 11) as i32) >> 23; // bits[20:12], sign-extended - let load = opc != 0; - let signed = opc >= 2; - let to_x = size == 3 || opc == 2; // x-sized access, or sign-extend to 64-bit - let suffix: &str = match size { - 0 => "b", - 1 => "h", - 2 if signed => "w", - _ => "", - }; - out.push_str(if load { "ldur" } else { "stur" }); - if signed { - out.push('s'); - } - out.push_str(suffix); - out.push(' '); - gp(out, to_x, rd(w)); - out.push_str(", ["); - sp(out, true, rn(w)); - if imm9 != 0 { - let _ = write!(out, ", #{imm9}]"); - } else { - out.push(']'); - } - return; - } - - // STUR/LDUR (FP/SIMD), unscaled signed 9-bit offset. Width is b/h/s/d by - // size, or q (128-bit) when opc bit1 is set. - if w & 0x3F20_0C00 == 0x3C00_0000 { - let size = w >> 30; - let opc = (w >> 22) & 3; - let imm9: i32 = ((w << 11) as i32) >> 23; - let load = opc & 1 != 0; - let c: char = if opc & 2 != 0 { - 'q' - } else { - match size { - 0 => 'b', - 1 => 'h', - 2 => 's', - _ => 'd', - } - }; - out.push_str(if load { "ldur " } else { "stur " }); - let _ = write!(out, "{c}{}, [", rd(w)); - sp(out, true, rn(w)); - if imm9 != 0 { - let _ = write!(out, ", #{imm9}]"); - } else { - out.push(']'); - } - return; - } - - // umulh / smulh (unsigned/signed 64x64 -> high 64). Ra field is 31. - if w & 0xFFE0_FC00 == 0x9BC0_7C00 || w & 0xFFE0_FC00 == 0x9B40_7C00 { - let uns = (w >> 23) & 1 != 0; - let _ = write!( - out, - "{} x{}, x{}, x{}", - if uns { "umulh" } else { "smulh" }, - rd(w), - rn(w), - rm(w) - ); - return; - } - - // Bitfield move (SBFM/UBFM/BFM) with the usual aliases. - if w & 0x1F80_0000 == 0x1300_0000 && (w >> 29) & 3 != 3 { - bitfield(out, w); - return; - } - - // stp/ldp signed offset (bit31 selects 32/64-bit, bit22 selects load). - if w & 0x7FC0_0000 == 0x2900_0000 || w & 0x7FC0_0000 == 0x2940_0000 { - let x64 = (w >> 31) & 1 != 0; - let load = (w >> 22) & 1 != 0; - let scale: u32 = if x64 { 3 } else { 2 }; - let imm7: i32 = (((w << 10) as i32) >> 25) << scale; // bits[21:15], scaled - out.push_str(if load { "ldp " } else { "stp " }); - gp(out, x64, rd(w)); - out.push_str(", "); - gp(out, x64, ra(w)); // rt2 is bits[14:10] - out.push_str(", ["); - sp(out, true, rn(w)); - if imm7 != 0 { - let _ = write!(out, ", #{imm7}]"); - } else { - out.push(']'); - } - return; - } - - // adr / adrp Rd, imm (PC-relative address; adrp is page-scaled). - if w & 0x1F00_0000 == 0x1000_0000 { - let page = (w >> 31) & 1 != 0; - let immlo: i64 = ((w >> 29) & 3) as i64; - let immhi: i64 = ((((w & 0x00FF_FFE0) << 8) as i32) >> 13) as i64; // bits[23:5], sign-extended - let mut imm: i64 = (immhi << 2) | immlo; - if page { - imm <<= 12; - } - out.push_str(if page { "adrp " } else { "adr " }); - gp(out, true, rd(w)); - if let Some(c) = ctx { - // absolute target: adrp is page-based, adr is byte-based - let pc_base: u64 = if page { c.pc & !0xFFF } else { c.pc }; - let abs: u64 = pc_base.wrapping_add(imm as u64); - let _ = write!(out, ", 0x{abs:x}"); - annotate_sym(out, c.syms, abs); - } else if imm < 0 { - let _ = write!(out, ", .-0x{:x}", (-imm) as u64); - } else { - let _ = write!(out, ", .+0x{imm:x}"); - } - return; - } - - if ordered_mem(out, w) { - return; - } - if mem_imm(out, w) { - return; - } - if fp_decode(out, w) { - return; - } - if vec_decode(out, w) { - return; - } - - // Unknown: emit the raw word so nothing is silently lost. - let _ = write!(out, ".word 0x{w:08x}"); -} - -/// Decode a logical-immediate `(N:immr:imms)` to its 32/64-bit value, or -/// `None` if the encoding is reserved. This is ARM's `DecodeBitMasks` -/// (immediate result only): a run of `s+1` ones, rotated right by `r` within -/// an `esize`-bit element, replicated across the register. -fn decode_bitmask(x64: bool, n: u32, imms: u32, immr: u32) -> Option { - let width: u32 = if x64 { 64 } else { 32 }; - // len = position of the highest set bit of (N : NOT(imms)), a 7-bit quantity. - let combined: u32 = ((n & 1) << 6) | ((!imms) & 0x3F); - if combined == 0 { - return None; - } - let mut len: u32 = 6; - while len > 0 && (combined & (1 << len)) == 0 { - len -= 1; - } - if len == 0 { - return None; // element size 1 is not a valid logical immediate - } - let esize: u32 = 1 << len; - if esize > width { - return None; // N == 1 is only valid for the 64-bit form - } - let levels: u32 = esize - 1; - let s: u32 = imms & levels; - let r: u32 = immr & levels; - if s == levels { - return None; // an all-ones element is reserved - } - let elem: u64 = (1u64 << (s + 1)) - 1; - let rotated = ror_in_esize(elem, r, esize); - let mut result: u64 = 0; - let mut pos: u32 = 0; - while pos < width { - result |= rotated << pos; - pos += esize; - } - Some(if x64 { result } else { result & 0xFFFF_FFFF }) -} - -/// Rotate the low `esize` bits of `v` right by `r`. -fn ror_in_esize(v: u64, r: u32, esize: u32) -> u64 { - let mask: u64 = if esize == 64 { - u64::MAX - } else { - (1u64 << esize) - 1 - }; - let rr = r % esize; - if rr == 0 { - return v & mask; - } - ((v >> rr) | (v << (esize - rr))) & mask -} - -/// Load-acquire / store-release register (`ldar{,b,h}`, `stlr{,b,h}`): `[xn]` addressing only. -fn ordered_mem(out: &mut String, w: u32) -> bool { - let mnem = match w & 0x3FFF_FC00 { - 0x08DF_FC00 => "ldar", - 0x089F_FC00 => "stlr", - _ => return false, - }; - let size = w >> 30; - out.push_str(mnem); - out.push_str(["b", "h", "", ""][size as usize]); - out.push(' '); - gp(out, size == 3, rd(w)); - out.push_str(", ["); - sp(out, true, rn(w)); - out.push(']'); - true -} - -/// Scaled-unsigned-offset loads/stores (integer and FP). Returns whether it -/// matched. -fn mem_imm(out: &mut String, w: u32) -> bool { - struct Entry { - match_word: u32, - mnem: &'static str, - x64: bool, - scale: u32, - fp: bool, - dbl: bool, - } - const TABLE: &[Entry] = &[ - Entry { - match_word: 0xF900_0000, - mnem: "str", - x64: true, - scale: 3, - fp: false, - dbl: false, - }, - Entry { - match_word: 0xF940_0000, - mnem: "ldr", - x64: true, - scale: 3, - fp: false, - dbl: false, - }, - Entry { - match_word: 0xB900_0000, - mnem: "str", - x64: false, - scale: 2, - fp: false, - dbl: false, - }, - Entry { - match_word: 0xB940_0000, - mnem: "ldr", - x64: false, - scale: 2, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x3900_0000, - mnem: "strb", - x64: false, - scale: 0, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x3940_0000, - mnem: "ldrb", - x64: false, - scale: 0, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x39C0_0000, - mnem: "ldrsb", - x64: false, - scale: 0, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x7900_0000, - mnem: "strh", - x64: false, - scale: 1, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x7940_0000, - mnem: "ldrh", - x64: false, - scale: 1, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x79C0_0000, - mnem: "ldrsh", - x64: false, - scale: 1, - fp: false, - dbl: false, - }, - Entry { - match_word: 0xBD00_0000, - mnem: "str", - x64: false, - scale: 2, - fp: true, - dbl: false, - }, - Entry { - match_word: 0xBD40_0000, - mnem: "ldr", - x64: false, - scale: 2, - fp: true, - dbl: false, - }, - Entry { - match_word: 0xFD00_0000, - mnem: "str", - x64: false, - scale: 3, - fp: true, - dbl: true, - }, - Entry { - match_word: 0xFD40_0000, - mnem: "ldr", - x64: false, - scale: 3, - fp: true, - dbl: true, - }, - Entry { - match_word: 0x3D80_0000, - mnem: "str", - x64: false, - scale: 4, - fp: true, - dbl: false, - }, // Q - Entry { - match_word: 0x3DC0_0000, - mnem: "ldr", - x64: false, - scale: 4, - fp: true, - dbl: false, - }, // Q - Entry { - match_word: 0x7D00_0000, - mnem: "str", - x64: false, - scale: 1, - fp: true, - dbl: false, - }, // H (f16) - Entry { - match_word: 0x7D40_0000, - mnem: "ldr", - x64: false, - scale: 1, - fp: true, - dbl: false, - }, // H (f16) - ]; - for m in TABLE { - if w & 0xFFC0_0000 != m.match_word { - continue; - } - let off: u32 = ((w >> 10) & 0xFFF) << m.scale; - out.push_str(m.mnem); - out.push(' '); - if m.fp { - // The FP view: Q (128-bit), D (64-bit), H (16-bit, the f16 - // boundary form), else S. - let c: char = if m.scale == 4 { - 'q' - } else if m.dbl { - 'd' - } else if m.scale == 1 { - 'h' - } else { - 's' - }; - let _ = write!(out, "{c}{}", rd(w)); - } else { - gp(out, m.x64, rd(w)); - } - out.push_str(", ["); - sp(out, true, rn(w)); - if off != 0 { - let _ = write!(out, ", #{off}]"); - } else { - out.push(']'); - } - return true; - } - // strb/ldrb/etc with a zero-offset `[xn]` form already covered (off field 0). - false -} - -/// Scalar floating-point instructions. Returns whether it matched. -fn fp_decode(out: &mut String, w: u32) -> bool { - let dbl = (w >> 22) & 1 != 0; - let fc: char = if dbl { 'd' } else { 's' }; - // Three-register scalar FP arithmetic. - const FP3: &[(u32, &str)] = &[ - (0x1E20_2800, "fadd"), - (0x1E20_3800, "fsub"), - (0x1E20_0800, "fmul"), - (0x1E20_1800, "fdiv"), - ]; - for &(m, mnem) in FP3 { - if w & 0xFFA0_FC00 == m { - let _ = write!(out, "{mnem} {fc}{}, {fc}{}, {fc}{}", rd(w), rn(w), rm(w)); - return true; - } - } - // Floating-point data-processing (3-source): Rd = op(Rn*Rm, Ra). Mask - // excludes ftype (bit 22, read via `dbl` above) and the four 5-bit - // register fields. - const FP3A: &[(u32, &str)] = &[ - (0x1F00_0000, "fmadd"), - (0x1F00_8000, "fmsub"), - (0x1F20_0000, "fnmadd"), - (0x1F20_8000, "fnmsub"), - ]; - for &(m, mnem) in FP3A { - if w & 0xFF20_8000 == m { - let _ = write!( - out, - "{mnem} {fc}{}, {fc}{}, {fc}{}, {fc}{}", - rd(w), - rn(w), - rm(w), - ra(w) - ); - return true; - } - } - if w & 0xFFA0_0C00 == 0x1E20_0C00 { - // fcsel (mask excludes the cond field [12:15]) - let _ = write!( - out, - "fcsel {fc}{}, {fc}{}, {fc}{}, {}", - rd(w), - rn(w), - rm(w), - cond_name((w >> 12) & 0xF) - ); - return true; - } - if w & 0xFFA0_FC1F == 0x1E20_2000 { - // fcmp - let _ = write!(out, "fcmp {fc}{}, {fc}{}", rn(w), rm(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E60_4000 { - // fmov (reg, 64-bit view) - let _ = write!(out, "fmov d{}, d{}", rd(w), rn(w)); - return true; - } - // fmov between a general register and an FP register (both directions, - // s/d). - struct Fmov { - match_word: u32, - to_gpr: bool, - x64: bool, - } - const FMOV: &[Fmov] = &[ - Fmov { - match_word: 0x1E27_0000, - to_gpr: false, - x64: false, - }, - Fmov { - match_word: 0x9E67_0000, - to_gpr: false, - x64: true, - }, - Fmov { - match_word: 0x1E26_0000, - to_gpr: true, - x64: false, - }, - Fmov { - match_word: 0x9E66_0000, - to_gpr: true, - x64: true, - }, - ]; - for e in FMOV { - if w & 0xFFFF_FC00 == e.match_word { - let f: char = if e.x64 { 'd' } else { 's' }; - out.push_str("fmov "); - if e.to_gpr { - gp(out, e.x64, rd(w)); - let _ = write!(out, ", {f}{}", rn(w)); - } else { - let _ = write!(out, "{f}{}, ", rd(w)); - gp(out, e.x64, rn(w)); - } - return true; - } - } - // Two-register scalar FP ops (sqrt, rounding, conversions). - const FP2: &[(u32, &str)] = &[ - (0x1E21_C000, "fsqrt"), - (0x1E24_4000, "frintn"), - (0x1E24_C000, "frintp"), - (0x1E25_4000, "frintm"), - (0x1E25_C000, "frintz"), - ]; - for &(m, mnem) in FP2 { - if w & 0xFFBF_FC00 == m { - let _ = write!(out, "{mnem} {fc}{}, {fc}{}", rd(w), rn(w)); - return true; - } - } - if w & 0xFFBF_FC00 == 0x1E22_0000 { - // scvtf sd, wn - let _ = write!(out, "scvtf {fc}{}, ", rd(w)); - gp(out, false, rn(w)); - return true; - } - if w & 0xFFBF_FC00 == 0x1E23_0000 { - // ucvtf - let _ = write!(out, "ucvtf {fc}{}, ", rd(w)); - gp(out, false, rn(w)); - return true; - } - if w & 0xFFBF_FC00 == 0x1E38_0000 { - // fcvtzs wd, sn - out.push_str("fcvtzs "); - gp(out, false, rd(w)); - let _ = write!(out, ", {fc}{}", rn(w)); - return true; - } - if w & 0xFFBF_FC00 == 0x1E39_0000 { - // fcvtzu - out.push_str("fcvtzu "); - gp(out, false, rd(w)); - let _ = write!(out, ", {fc}{}", rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E22_C000 { - // fcvt d, s (s->d) - let _ = write!(out, "fcvt d{}, s{}", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E62_4000 { - // fcvt s, d (d->s) - let _ = write!(out, "fcvt s{}, d{}", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1EE2_4000 { - // fcvt s, h (h->s, f16 widen) - let _ = write!(out, "fcvt s{}, h{}", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E23_C000 { - // fcvt h, s (s->h, f16 narrow) - let _ = write!(out, "fcvt h{}, s{}", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E63_C000 { - // fcvt h, d (d->h, f16 narrow, single round) - let _ = write!(out, "fcvt h{}, d{}", rd(w), rn(w)); - return true; - } - false -} - -/// NEON (128-bit vector) instructions. Returns whether it matched. -fn vec_decode(out: &mut String, w: u32) -> bool { - // Three-register 4S / 16B forms. - struct V3 { - match_word: u32, - mnem: &'static str, - b16: bool, - } - const V3_TABLE: &[V3] = &[ - V3 { - match_word: 0x4E20_D400, - mnem: "fadd", - b16: false, - }, - V3 { - match_word: 0x4EA0_D400, - mnem: "fsub", - b16: false, - }, - V3 { - match_word: 0x6E20_DC00, - mnem: "fmul", - b16: false, - }, - V3 { - match_word: 0x6E20_FC00, - mnem: "fdiv", - b16: false, - }, - V3 { - match_word: 0x4EA0_F400, - mnem: "fmin", - b16: false, - }, - V3 { - match_word: 0x4E20_F400, - mnem: "fmax", - b16: false, - }, - V3 { - match_word: 0x4E20_E400, - mnem: "fcmeq", - b16: false, - }, - V3 { - match_word: 0x6EA0_E400, - mnem: "fcmgt", - b16: false, - }, - V3 { - match_word: 0x6E20_E400, - mnem: "fcmge", - b16: false, - }, - V3 { - match_word: 0x6E60_1C00, - mnem: "bsl", - b16: true, - }, - V3 { - match_word: 0x4E20_CC00, - mnem: "fmla", - b16: false, - }, - V3 { - match_word: 0x4EA0_CC00, - mnem: "fmls", - b16: false, - }, - V3 { - match_word: 0x6E20_D400, - mnem: "faddp", - b16: false, - }, - ]; - for e in V3_TABLE { - if w == (e.match_word | (rm(w) << 16) | (rn(w) << 5) | rd(w)) { - let s = if e.b16 { "16b" } else { "4s" }; - let _ = write!( - out, - "{} v{}.{s}, v{}.{s}, v{}.{s}", - e.mnem, - rd(w), - rn(w), - rm(w) - ); - return true; - } - } - // Two-register vector forms. - if w & 0xFFFF_FC00 == 0x6EA0_F800 { - return vec2(out, w, "fneg", "4s"); - } - if w & 0xFFFF_FC00 == 0x6EA1_F800 { - return vec2(out, w, "fsqrt", "4s"); - } - if w & 0xFFFF_FC00 == 0x6E20_5800 { - return vec2(out, w, "mvn", "16b"); - } - // Horizontal reduce: addv sums the 4S lanes to a scalar S register; - // faddpScalar finishes a pairwise float reduction from a 2S pair. Both - // cross from the Vn.4S/2S view into Sd. - if w & 0xFFFF_FC00 == 0x4EB1_B800 { - let _ = write!(out, "addv s{}, v{}.4s", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x7E30_D800 { - let _ = write!(out, "faddp s{}, v{}.2s", rd(w), rn(w)); - return true; - } - // mov vd.16b, vn.16b is orr with rm == rn. - if w & 0xFFE0_FC00 == 0x4EA0_1C00 && rm(w) == rn(w) { - let _ = write!(out, "mov v{}.16b, v{}.16b", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x4E04_0C00 { - // dup Vd.4S, Wn - let _ = write!(out, "dup v{}.4s, ", rd(w)); - gp(out, false, rn(w)); - return true; - } - // dup Vd.4S, Vn.s[i] / dup Sd, Vn.s[i] / ins Vd.s[i], Vn.s[0]: imm5 in - // bits[16:20]. - let imm5 = (w >> 16) & 0x1F; - let lane = (imm5 >> 3) & 3; - if w & 0xFFE0_FC00 == 0x4E00_0400 { - let _ = write!(out, "dup v{}.4s, v{}.s[{lane}]", rd(w), rn(w)); - return true; - } - if w & 0xFFE0_FC00 == 0x5E00_0400 { - let _ = write!(out, "dup s{}, v{}.s[{lane}]", rd(w), rn(w)); - return true; - } - if w & 0xFFE0_8C00 == 0x6E00_0400 { - let _ = write!(out, "ins v{}.s[{lane}], v{}.s[0]", rd(w), rn(w)); - return true; - } - if w & 0xFFE0_FC00 == 0x0E00_3C00 { - // umov Wd, Vn.s[i] - out.push_str("umov "); - gp(out, false, rd(w)); - let _ = write!(out, ", v{}.s[{lane}]", rn(w)); - return true; - } - if w & 0xFFE0_FC00 == 0x4E00_1C00 { - // ins Vd.s[i], Wn - let _ = write!(out, "ins v{}.s[{lane}], ", rd(w)); - gp(out, false, rn(w)); - return true; - } - false -} - -fn vec2(out: &mut String, w: u32, mnem: &str, suffix: &str) -> bool { - let _ = write!(out, "{mnem} v{}.{suffix}, v{}.{suffix}", rd(w), rn(w)); - true -} - -/// Render a bitfield-move (SBFM/UBFM/BFM) using the alias a debugger expects -/// (uxt*/sxt*/lsl/lsr/asr/ubfx/sbfx/bfi/bfxil). `opc` selects the family; -/// `immr`/`imms` drive the alias choice. -fn bitfield(out: &mut String, w: u32) { - let opc = (w >> 29) & 3; - let x64 = sf(w); - let width: u32 = if x64 { 64 } else { 32 }; - let immr: u32 = (w >> 16) & 0x3F; - let imms: u32 = (w >> 10) & 0x3F; - - match opc { - 2 => { - // UBFM - if imms + 1 == immr && imms != width - 1 { - // lsl #(width-1-imms) - bf_two(out, "lsl", x64, rd(w), rn(w)); - let _ = write!(out, ", #{}", width - 1 - imms); - } else if imms == width - 1 { - // lsr #immr - bf_two(out, "lsr", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}"); - } else if immr == 0 && imms == 7 && !x64 { - bf_two(out, "uxtb", false, rd(w), rn(w)); - } else if immr == 0 && imms == 15 && !x64 { - bf_two(out, "uxth", false, rd(w), rn(w)); - } else if imms < immr { - // ubfiz Rd, Rn, #(width-immr), #(imms+1) - bf_two(out, "ubfiz", x64, rd(w), rn(w)); - let _ = write!(out, ", #{}, #{}", width - immr, imms + 1); - } else { - // ubfx Rd, Rn, #immr, #(imms-immr+1) - bf_two(out, "ubfx", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}, #{}", imms - immr + 1); - } - } - 0 => { - // SBFM - if imms == width - 1 { - // asr #immr - bf_two(out, "asr", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}"); - } else if immr == 0 && imms == 7 { - bf_two(out, "sxtb", x64, rd(w), rn(w)); - } else if immr == 0 && imms == 15 { - bf_two(out, "sxth", x64, rd(w), rn(w)); - } else if immr == 0 && imms == 31 && x64 { - // sxtw Xd, Wn - out.push_str("sxtw "); - gp(out, true, rd(w)); - out.push_str(", "); - gp(out, false, rn(w)); - } else if imms < immr { - // sbfiz - bf_two(out, "sbfiz", x64, rd(w), rn(w)); - let _ = write!(out, ", #{}, #{}", width - immr, imms + 1); - } else { - bf_two(out, "sbfx", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}, #{}", imms - immr + 1); - } - } - _ => { - // BFM - if imms < immr { - // bfi Rd, Rn, #(width-immr), #(imms+1) - bf_two(out, "bfi", x64, rd(w), rn(w)); - let _ = write!(out, ", #{}, #{}", width - immr, imms + 1); - } else { - // bfxil Rd, Rn, #immr, #(imms-immr+1) - bf_two(out, "bfxil", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}, #{}", imms - immr + 1); - } - } - } -} - -/// The common `mnem Rd, Rn` shape for bitfield aliases, at the register width. -fn bf_two(out: &mut String, mnem: &str, xw: bool, d: u32, n: u32) { - out.push_str(mnem); - out.push(' '); - gp(out, xw, d); - out.push_str(", "); - gp(out, xw, n); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn round_trips_basic_words() { - // Hand-computed words (no dependency on the encoder module): - // ret = 0xD65F0000 | (30 << 5); add w0,w1,w2; bl .+8; nop. - assert_eq!(one(0xD65F_03C0), "ret"); - assert_eq!(one(0x0B02_0020), "add w0, w1, w2"); - assert_eq!(one(0x8B02_0020), "add x0, x1, x2"); - assert_eq!(one(0x9400_0002), "bl .+8"); - assert_eq!(one(0x1400_0002), "b .+8"); - // nop is outside the emitted-subset decode coverage: it must surface - // as a raw word, never a mis-decode. - assert_eq!(one(0xD503_201F), ".word 0xd503201f"); - // ldar/stlr in every width (GNU as output). - assert_eq!(one(0x88df_fc20), "ldar w0, [x1]"); - assert_eq!(one(0xc8df_fc62), "ldar x2, [x3]"); - assert_eq!(one(0x08df_fca4), "ldarb w4, [x5]"); - assert_eq!(one(0x48df_fce6), "ldarh w6, [x7]"); - assert_eq!(one(0x889f_fc20), "stlr w0, [x1]"); - assert_eq!(one(0xc89f_fc62), "stlr x2, [x3]"); - assert_eq!(one(0x089f_fca4), "stlrb w4, [x5]"); - assert_eq!(one(0x489f_fce6), "stlrh w6, [x7]"); - } - - #[test] - fn decodes_objdump_vectors() { - // Spot checks from the Zig suite (verified against objdump there). - assert_eq!(one(0xEB09_0108), "subs x8, x8, x9"); - assert_eq!(one(0x3400_00E8), "cbz w8, .+28"); - assert_eq!(one(0x5400_00C3), "b.cc .+24"); - assert_eq!(one(0xF85F_83A8), "ldur x8, [x29, #-8]"); - assert_eq!(one(0xA941_7BFD), "ldp x29, x30, [sp, #16]"); - assert_eq!(one(0x5300_3D08), "uxth w8, w8"); - assert_eq!(one(0x531F_7908), "lsl w8, w8, #1"); - assert_eq!(one(0x9340_7C00), "sxtw x0, w0"); - assert_eq!(one(0x9280_0AA1), "mov x1, #0xffffffffffffffaa"); - } - - #[test] - fn format_with_lines_interleaves_markers() { - let code = [0x0B02_0020, 0x1B01_7C00, 0xD65F_03C0]; - let lines = [ - SourceLine { offset: 0, line: 2 }, - SourceLine { offset: 4, line: 3 }, - ]; - assert_eq!( - format_with_lines(&code, &lines), - "; line 2\n\ - 0000: 0b020020 add w0, w1, w2\n\ - ; line 3\n\ - 0004: 1b017c00 mul w0, w0, w1\n\ - 0008: d65f03c0 ret\n" - ); - } - - #[test] - fn format_module_annotates_branch_targets() { - // fn f at word 0, fn g at word 3; b .+8 lands at words 2 and 4. - let code = [ - 0x1400_0002, - 0xD65F_03C0, - 0x1400_0002, - 0xD65F_03C0, - 0xD65F_03C0, - ]; - let syms = [Sym { name: "f", word: 0 }, Sym { name: "g", word: 3 }]; - let text = format_module(&code, &syms); - assert!(text.contains("b .+8 "), "{text}"); - assert!(text.contains("b .+8 "), "{text}"); - } - - #[test] - fn format_elf_resolves_absolute_targets() { - let base: u64 = 0x1000; - let code = [0x9400_0002, 0xB000_0000, 0xD65F_03C0]; - let syms = [ - AddrSym { - name: "start", - addr: 0x1000, - size: 8, - is_func: true, - }, - AddrSym { - name: "callee", - addr: 0x1008, - size: 4, - is_func: true, - }, - AddrSym { - name: "gv", - addr: 0x2000, - size: 16, - is_func: false, - }, - ]; - let text = format_elf(&code, base, &syms); - assert!(text.contains("0000000000001000 :"), "{text}"); - assert!(text.contains("0000000000001008 :"), "{text}"); - assert!(text.contains("bl 0x1008 "), "{text}"); - assert!(text.contains("adrp x0, 0x2000 "), "{text}"); - } -} diff --git a/src/target/src/aarch64/isel_tests.rs b/src/target/src/aarch64/isel_tests.rs index 7281c34..f832d2a 100644 --- a/src/target/src/aarch64/isel_tests.rs +++ b/src/target/src/aarch64/isel_tests.rs @@ -330,7 +330,7 @@ fn listing(f: &Function) -> Vec { compile(f) .unwrap() .iter() - .map(|&w| crate::aarch64::disasm::one(w)) + .map(|&w| crate::aarch64::listing::one(w)) .collect() } diff --git a/src/target/src/aarch64/listing.rs b/src/target/src/aarch64/listing.rs new file mode 100644 index 0000000..c750869 --- /dev/null +++ b/src/target/src/aarch64/listing.rs @@ -0,0 +1,272 @@ +//! Assembly listings of AArch64 code buffers, for debugging code generation. +//! +//! The text of each instruction comes from `volt_isa_aarch64::format`; this module +//! only lays words out with addresses, labels, source-line markers and symbol +//! annotations. A word the decoder does not recognise prints as `.word 0x`. + +use alloc::string::{String, ToString as _}; +use core::fmt::Write as _; +use volt_isa_aarch64::flow::flow; +use volt_isa_aarch64::format::{self, Symbols}; + +/// A source-line-table row: the byte offset where a source line's code begins. +pub struct SourceLine { + pub offset: u32, + pub line: u32, +} + +/// A function's location in a linked image: its name and its word offset. +pub struct Sym<'a> { + pub name: &'a str, + pub word: usize, +} + +/// A defined symbol by absolute address (a function or data object), for +/// resolving branch and `adrp` targets when listing an ELF image. +pub struct AddrSym<'a> { + pub name: &'a str, + pub addr: u64, + pub size: u64, + pub is_func: bool, +} + +/// A decoded source-line row keyed by absolute address (from a `.debug_line` +/// program), for objdump `-S`-style annotation of a real ELF. +pub struct AddrLine { + pub addr: u64, + pub line: u32, +} + +/// The symbol containing an address: the greatest-addressed symbol at or before it. +struct AddrSyms<'a>(&'a [AddrSym<'a>]); + +impl Symbols for AddrSyms<'_> { + fn lookup(&self, addr: u64) -> Option<(&str, u64)> { + let mut best: Option<&AddrSym<'_>> = None; + for s in self.0 { + if s.addr <= addr && best.is_none_or(|b| s.addr > b.addr) { + best = Some(s); + } + } + best.map(|s| (s.name, addr - s.addr)) + } +} + +/// Render one instruction word as `mnemonic operands`. +pub fn one(word: u32) -> String { + format::word(word).to_string() +} + +/// Render a whole instruction stream (one word per line, `addr: hex text`). +pub fn format(code: &[u32]) -> String { + let mut out = String::new(); + for (i, &word) in code.iter().enumerate() { + let _ = writeln!(out, "{:04x}: {word:08x} {}", i * 4, format::word(word)); + } + out +} + +/// Render a listing with source-line markers interleaved (objdump `-S` +/// style): a `; line N` header precedes the instructions belonging to that +/// source line. `lines` must be sorted by offset (as isel produces). +pub fn format_with_lines(code: &[u32], lines: &[SourceLine]) -> String { + let mut out = String::new(); + let mut next = 0; + for (i, &word) in code.iter().enumerate() { + let offset = (i * 4) as u32; + while next < lines.len() && lines[next].offset == offset { + let _ = writeln!(out, "; line {}", lines[next].line); + next += 1; + } + let _ = writeln!(out, "{offset:04x}: {word:08x} {}", format::word(word)); + } + out +} + +/// Render a whole linked module: each function gets a `name:` label at its +/// offset, and every PC-relative branch is annotated with the function it lands +/// in (`` or ``), so a linked image reads like a symbolized +/// listing. +pub fn format_module(code: &[u32], syms: &[Sym<'_>]) -> String { + let mut out = String::new(); + for (i, &word) in code.iter().enumerate() { + for s in syms.iter().filter(|s| s.word == i) { + if i != 0 { + out.push('\n'); + } + let _ = writeln!(out, "{}:", s.name); + } + let _ = write!(out, "{:04x}: {word:08x} {}", i * 4, format::word(word)); + let target = flow(word, (i * 4) as u64) + .target() + .filter(|&t| (t as i64) >= 0) + .map(|t| (t / 4) as usize); + if let Some(target) = target + && let Some(s) = syms + .iter() + .filter(|s| s.word <= target) + .max_by_key(|s| s.word) + { + if target == s.word { + let _ = write!(out, " <{}>", s.name); + } else { + let _ = write!(out, " <{}+0x{:x}>", s.name, (target - s.word) * 4); + } + } + out.push('\n'); + } + out +} + +/// Render an ELF code section as an objdump-style listing: absolute +/// addresses, function labels, and branch/`adrp` targets resolved to +/// `0x `. `base` is the section's load address; `syms` are all +/// defined symbols (functions and data). +pub fn format_elf(code: &[u32], base: u64, syms: &[AddrSym<'_>]) -> String { + format_elf_with_lines(code, base, syms, &[]) +} + +/// Like [`format_elf`], but interleaves `; line N` markers from `lines` +/// (sorted by address) before the instruction at each address, so a listing +/// of a `-g` object reads with its source lines. +pub fn format_elf_with_lines( + code: &[u32], + base: u64, + syms: &[AddrSym<'_>], + lines: &[AddrLine], +) -> String { + let symbols = AddrSyms(syms); + let mut out = String::new(); + let mut next = 0; + for (i, &word) in code.iter().enumerate() { + let addr = base + i as u64 * 4; + for s in syms.iter().filter(|s| s.is_func && s.addr == addr) { + if i != 0 { + out.push('\n'); + } + let _ = writeln!(out, "{addr:016x} <{}>:", s.name); + } + while next < lines.len() && lines[next].addr < addr { + next += 1; // skip any before this insn + } + while next < lines.len() && lines[next].addr == addr { + let _ = writeln!(out, "; line {}", lines[next].line); + next += 1; + } + let _ = writeln!( + out, + "{addr:08x}: {word:08x} {}", + format::word_with(word, addr, &symbols) + ); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips_basic_words() { + // Hand-computed words (no dependency on the encoder module): + // ret = 0xD65F0000 | (30 << 5); add w0,w1,w2; bl .+8; nop. + assert_eq!(one(0xD65F_03C0), "ret"); + assert_eq!(one(0x0B02_0020), "add w0, w1, w2"); + assert_eq!(one(0x8B02_0020), "add x0, x1, x2"); + assert_eq!(one(0x9400_0002), "bl .+8"); + assert_eq!(one(0x1400_0002), "b .+8"); + assert_eq!(one(0xD503_201F), "nop"); + // A word the decoder does not recognise surfaces as a raw word. + assert_eq!( + one(0x0e21_4022 /* addhn: not decoded */), + ".word 0x0e214022" + ); + // ldar/stlr in every width (GNU as output). + assert_eq!(one(0x88df_fc20), "ldar w0, [x1]"); + assert_eq!(one(0xc8df_fc62), "ldar x2, [x3]"); + assert_eq!(one(0x08df_fca4), "ldarb w4, [x5]"); + assert_eq!(one(0x48df_fce6), "ldarh w6, [x7]"); + assert_eq!(one(0x889f_fc20), "stlr w0, [x1]"); + assert_eq!(one(0xc89f_fc62), "stlr x2, [x3]"); + assert_eq!(one(0x089f_fca4), "stlrb w4, [x5]"); + assert_eq!(one(0x489f_fce6), "stlrh w6, [x7]"); + } + + #[test] + fn decodes_objdump_vectors() { + // Spot checks from the Zig suite (verified against objdump there). + assert_eq!(one(0xEB09_0108), "subs x8, x8, x9"); + assert_eq!(one(0x3400_00E8), "cbz w8, .+28"); + assert_eq!(one(0x5400_00C3), "b.cc .+24"); + assert_eq!(one(0xF85F_83A8), "ldur x8, [x29, #-8]"); + assert_eq!(one(0xA941_7BFD), "ldp x29, x30, [sp, #16]"); + assert_eq!(one(0x5300_3D08), "uxth w8, w8"); + assert_eq!(one(0x531F_7908), "lsl w8, w8, #1"); + assert_eq!(one(0x9340_7C00), "sxtw x0, w0"); + assert_eq!(one(0x9280_0AA1), "mov x1, #0xffffffffffffffaa"); + } + + #[test] + fn format_with_lines_interleaves_markers() { + let code = [0x0B02_0020, 0x1B01_7C00, 0xD65F_03C0]; + let lines = [ + SourceLine { offset: 0, line: 2 }, + SourceLine { offset: 4, line: 3 }, + ]; + assert_eq!( + format_with_lines(&code, &lines), + "; line 2\n\ + 0000: 0b020020 add w0, w1, w2\n\ + ; line 3\n\ + 0004: 1b017c00 mul w0, w0, w1\n\ + 0008: d65f03c0 ret\n" + ); + } + + #[test] + fn format_module_annotates_branch_targets() { + // fn f at word 0, fn g at word 3; b .+8 lands at words 2 and 4. + let code = [ + 0x1400_0002, + 0xD65F_03C0, + 0x1400_0002, + 0xD65F_03C0, + 0xD65F_03C0, + ]; + let syms = [Sym { name: "f", word: 0 }, Sym { name: "g", word: 3 }]; + let text = format_module(&code, &syms); + assert!(text.contains("b .+8 "), "{text}"); + assert!(text.contains("b .+8 "), "{text}"); + } + + #[test] + fn format_elf_resolves_absolute_targets() { + let base: u64 = 0x1000; + let code = [0x9400_0002, 0xB000_0000, 0xD65F_03C0]; + let syms = [ + AddrSym { + name: "start", + addr: 0x1000, + size: 8, + is_func: true, + }, + AddrSym { + name: "callee", + addr: 0x1008, + size: 4, + is_func: true, + }, + AddrSym { + name: "gv", + addr: 0x2000, + size: 16, + is_func: false, + }, + ]; + let text = format_elf(&code, base, &syms); + assert!(text.contains("0000000000001000 :"), "{text}"); + assert!(text.contains("0000000000001008 :"), "{text}"); + assert!(text.contains("bl 0x1008 "), "{text}"); + assert!(text.contains("adrp x0, 0x2000 "), "{text}"); + } +} diff --git a/src/target/src/aarch64/mod.rs b/src/target/src/aarch64/mod.rs index e149dd7..3fcfa94 100644 --- a/src/target/src/aarch64/mod.rs +++ b/src/target/src/aarch64/mod.rs @@ -1,6 +1,4 @@ //! AArch64 encoding, Function-based native selection, object linking and JIT. -pub mod decode; -pub mod disasm; pub mod encode; pub mod isel; #[cfg(all( @@ -12,6 +10,6 @@ pub mod isel; ))] pub mod jit; pub mod link; +pub mod listing; pub mod object; pub mod peephole; -pub mod simd_struct; From 9f0c3227c4eab471e2106edf6d58f92470e2b55b Mon Sep 17 00:00:00 2001 From: Theo Paris Date: Tue, 13 Oct 2026 01:02:32 -0700 Subject: [PATCH 13/13] feat(emulator): add fw_cfg --- README.md | 8 + src/emulator/src/devices/fw_cfg.rs | 98 +++++ src/emulator/src/devices/mod.rs | 3 + src/emulator/src/devices/nor_flash.rs | 611 ++++++++++++++++++++++++++ src/emulator/src/devices/pl031.rs | 286 ++++++++++++ src/emulator/src/system/boot.rs | 2 +- src/emulator/src/system/fdt.rs | 116 ++++- src/emulator/src/system/machine.rs | 19 +- src/emulator/src/system/mod.rs | 50 ++- 9 files changed, 1173 insertions(+), 20 deletions(-) create mode 100644 src/emulator/src/devices/fw_cfg.rs create mode 100644 src/emulator/src/devices/nor_flash.rs create mode 100644 src/emulator/src/devices/pl031.rs diff --git a/README.md b/README.md index 1467f3a..4d3d3f6 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,14 @@ The disk form supplies `root=/dev/vda rw` by default. `--cpus`, `--idle`, `--fast`, and `--spin` control the virtual CPU count and WFI policy. Use `volt-boot --help` for the complete generated interface. +Firmware boots use QEMU `virt` firmware devices: GICv3, fw_cfg, PL031 RTC, PCI ECAM and +two CFI NOR banks. The second flash bank stores volatile UEFI variables; no state persists +between runs. + +```sh +cargo run --release --bin volt-boot -- --bios /path/to/QEMU_EFI.fd --timeout 20 +``` + ## XNU boot `volt-boot` also boots an XNU kernel. A Mach-O image is detected by its magic, so no diff --git a/src/emulator/src/devices/fw_cfg.rs b/src/emulator/src/devices/fw_cfg.rs new file mode 100644 index 0000000..460a446 --- /dev/null +++ b/src/emulator/src/devices/fw_cfg.rs @@ -0,0 +1,98 @@ +//! QEMU fw_cfg's traditional MMIO interface. DMA is not advertised. +use super::bus::Device; + +pub const BASE: u64 = 0x0902_0000; +// Exclude the DMA address register at offset 0x10 from the advertised window. +pub const LEN: u64 = 0x10; + +pub struct FwCfg { + selector: u16, + offset: usize, + ram_size: [u8; 8], +} + +impl FwCfg { + pub fn new(ram_size: u64) -> Self { + Self { + selector: 0, + offset: 0, + ram_size: ram_size.to_le_bytes(), + } + } + + fn item(&self) -> &[u8] { + match self.selector { + 0x00 => b"QEMU", + 0x01 => &[1, 0, 0, 0], // Traditional interface only; no DMA. + 0x03 => &self.ram_size, + 0x05 | 0x0f => &[1, 0], // Number and maximum number of CPUs. + 0x19 => &[0, 0, 0, 0], // Empty file directory (big-endian count). + _ => &[], + } + } +} + +impl Device for FwCfg { + fn read(&mut self, offset: u64, size: u8) -> u64 { + if offset != 0 || !matches!(size, 1 | 2 | 4 | 8) { + return 0; + } + let item = self.item(); + let mut value = 0; + for byte in 0..usize::from(size) { + let at = self.offset.saturating_add(byte); + value |= u64::from(item.get(at).copied().unwrap_or(0)) << (byte * 8); + } + self.offset = self.offset.saturating_add(usize::from(size)); + value + } + + fn write(&mut self, offset: u64, size: u8, value: u64) { + if offset == 8 && size == 2 { + // MMIO selectors are big-endian; Device values are little-endian. + self.selector = (value as u16).swap_bytes() & !0x4000; + self.offset = 0; + } + // The data interface is read-only, and there is no DMA register. + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stream_preserves_bytes_across_widths_and_reselection() { + let mut device = FwCfg::new(0x0102_0304_0506_0708); + assert_eq!(device.read(0, 1), u64::from(b'Q')); + assert_eq!(device.read(0, 2), u64::from(u16::from_le_bytes(*b"EM"))); + assert_eq!(device.read(0, 4), u64::from(b'U')); + assert_eq!(device.read(0, 8), 0); + device.write(8, 2, 0); + assert_eq!(device.read(0, 4), u64::from(u32::from_le_bytes(*b"QEMU"))); + device.write(8, 2, 0x0300); + assert_eq!(device.read(0, 8), 0x0102_0304_0506_0708); + device.write(8, 2, 0x0100); + assert_eq!(device.read(0, 4), 1); + // The obsolete write-channel flag does not select a different item. + device.write(8, 2, 0x0040); + device.write(0, 4, 0); + assert_eq!(device.read(0, 4), u64::from(u32::from_le_bytes(*b"QEMU"))); + } + + #[test] + fn absent_items_and_unsupported_registers_do_not_change_the_stream() { + let mut device = FwCfg::new(128 << 20); + device.write(8, 2, 0x1900); + assert_eq!(device.read(0, 4), 0); + device.write(8, 2, 0xffff); + assert_eq!(device.read(0, 8), 0); + device.write(8, 2, 0); + device.write(0, 4, 0); + device.write(8, 1, 1); + device.write(16, 8, 0); + assert_eq!(device.read(8, 2), 0); + assert_eq!(device.read(16, 8), 0); + assert_eq!(device.read(0, 4), u64::from(u32::from_le_bytes(*b"QEMU"))); + } +} diff --git a/src/emulator/src/devices/mod.rs b/src/emulator/src/devices/mod.rs index 60b57e3..eb520bd 100644 --- a/src/emulator/src/devices/mod.rs +++ b/src/emulator/src/devices/mod.rs @@ -1,9 +1,12 @@ pub mod aic; pub mod bus; +pub mod fw_cfg; pub mod gicv2; pub mod gicv3; +pub mod nor_flash; pub mod pci; pub mod pl011; +pub mod pl031; pub mod pmgr; pub mod s5l_uart; pub mod virtio_blk; diff --git a/src/emulator/src/devices/nor_flash.rs b/src/emulator/src/devices/nor_flash.rs new file mode 100644 index 0000000..46ece2a --- /dev/null +++ b/src/emulator/src/devices/nor_flash.rs @@ -0,0 +1,611 @@ +//! Volatile variable flash: the second QEMU virt bank, not the firmware ROM. +//! +//! The bank is two x16 Intel-command-set chips on a little-endian x32 bus. +//! Operations complete synchronously; storage and the write buffer are allocated +//! only when the device is constructed. Command decoding follows QEMU's bank-wide +//! low-byte commands, with status and identification repeated in both x16 lanes. + +use alloc::vec; +use alloc::vec::Vec; + +use super::bus::Device; + +pub const BASE: u64 = 0x0400_0000; +pub const LEN: u64 = 64 * 1024 * 1024; +pub const BLOCK_SIZE: u64 = 256 * 1024; + +const BLOCK_COUNT: usize = (LEN / BLOCK_SIZE) as usize; +// QEMU pflash_cfi01 advertises 2 KiB per chip, hence 4 KiB for the bank. +// EDK2's P30 driver uses a smaller, compatible 128-byte buffer. +const BUFFER_SIZE: usize = 4096; +const BUFFER_WORDS: usize = BUFFER_SIZE / 4; +const READY: u8 = 0x80; +const ERASE_ERROR: u8 = 0x20; +const PROGRAM_ERROR: u8 = 0x10; +const BLOCK_LOCKED: u8 = 0x02; +const SEQUENCE_ERROR: u8 = ERASE_ERROR | PROGRAM_ERROR; + +#[derive(Clone, Copy)] +enum ReadMode { + Array, + Status, + Id, + Cfi, +} + +#[derive(Clone, Copy)] +enum WriteState { + Idle, + Program, + Erase { + block: usize, + }, + Lock { + block: usize, + }, + BufferCount { + block: usize, + }, + BufferData { + block: usize, + remaining: usize, + window: Option, + }, + BufferConfirm, +} + +#[derive(Clone, Copy)] +struct BufferedWord { + offset: u32, + value: u32, +} + +pub struct NorFlash { + storage: Vec, + locked: [bool; BLOCK_COUNT], + mode: ReadMode, + state: WriteState, + status: u8, + buffer: [BufferedWord; BUFFER_WORDS], + buffered_words: usize, +} + +impl NorFlash { + /// Construct an erased, unlocked variable bank. Its contents are not persisted. + pub fn new() -> Self { + Self { + storage: vec![0xff; LEN as usize], + locked: [false; BLOCK_COUNT], + mode: ReadMode::Array, + state: WriteState::Idle, + status: READY, + buffer: [BufferedWord { + offset: 0, + value: 0, + }; BUFFER_WORDS], + buffered_words: 0, + } + } + + fn valid_access(offset: u64, size: u8) -> bool { + matches!(size, 1 | 2 | 4 | 8) + && offset + .checked_add(u64::from(size)) + .is_some_and(|end| end <= LEN) + } + + fn block(offset: usize) -> usize { + offset / BLOCK_SIZE as usize + } + + fn read_array(&mut self) { + self.mode = ReadMode::Array; + self.state = WriteState::Idle; + self.buffered_words = 0; + } + + fn fail(&mut self, bits: u8) { + self.status |= READY | bits; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + self.buffered_words = 0; + } + + fn id(&self, offset: usize) -> u16 { + // QEMU virt IDs, from hw/arm/virt.c. ID address 2 is block lock status. + match (offset / 4) & 0xff { + 0 => 0x89, + 1 => 0x18, + 2 => u16::from(self.locked[Self::block(offset)]), + _ => 0, + } + } + + fn cfi(address: usize) -> u8 { + // Geometry and Intel PRI table from QEMU hw/block/pflash_cfi01.c. + // The CFI geometry is per chip: 32 MiB, 256 blocks of 128 KiB each. + match address { + 0x10 => b'Q', + 0x11 => b'R', + 0x12 => b'Y', + 0x13 => 1, + 0x15 => 0x31, + 0x1b => 0x45, + 0x1c => 0x55, + 0x1f | 0x20 => 7, + 0x21 => 0x0a, + 0x23..=0x25 => 4, + 0x27 => 25, + 0x28 => 2, + 0x2a => 11, + 0x2c => 1, + 0x2d => 0xff, + 0x30 => 2, + 0x31 => b'P', + 0x32 => b'R', + 0x33 => b'I', + 0x34 => b'1', + 0x35 => b'0', + 0x3f => 1, + _ => 0, + } + } + + fn program(&mut self, offset: usize, size: u8, value: u64) { + if self.locked[Self::block(offset)] + || self.locked[Self::block(offset + usize::from(size) - 1)] + { + self.status |= PROGRAM_ERROR | BLOCK_LOCKED; + return; + } + for byte in 0..usize::from(size) { + let new = (value >> (byte * 8)) as u8; + let old = self.storage[offset + byte]; + if new & !old != 0 { + self.status |= PROGRAM_ERROR; + } + // Programming can clear bits only; even erroneous writes never erase. + self.storage[offset + byte] = old & new; + } + } + + fn command(&mut self, offset: usize, command: u8) { + self.state = WriteState::Idle; + match command { + // 0x00 and 0xf0 are QEMU's historical read-array/probe aliases. + 0x00 | 0xff | 0xf0 => self.read_array(), + 0x10 | 0x40 => { + self.mode = ReadMode::Status; + self.state = WriteState::Program; + } + 0x20 => { + self.mode = ReadMode::Status; + self.state = WriteState::Erase { + block: Self::block(offset), + }; + } + 0x50 => { + // Clear sticky errors, not the ready bit. Like QEMU, return to array. + self.status = READY; + self.read_array(); + } + 0x60 => { + self.mode = ReadMode::Status; + self.state = WriteState::Lock { + block: Self::block(offset), + }; + } + 0x70 => self.mode = ReadMode::Status, + 0x90 => self.mode = ReadMode::Id, + 0x98 => self.mode = ReadMode::Cfi, + 0xe8 => { + self.mode = ReadMode::Status; + self.buffered_words = 0; + self.state = WriteState::BufferCount { + block: Self::block(offset), + }; + } + _ => self.fail(SEQUENCE_ERROR), + } + } +} + +impl Default for NorFlash { + fn default() -> Self { + Self::new() + } +} + +impl Device for NorFlash { + fn read(&mut self, offset: u64, size: u8) -> u64 { + if !Self::valid_access(offset, size) { + return 0; + } + let offset = offset as usize; + let mut value = 0; + for byte in 0..usize::from(size) { + let at = offset + byte; + let data = match self.mode { + ReadMode::Array => self.storage[at], + ReadMode::Status => { + if at & 1 == 0 { + self.status + } else { + 0 + } + } + ReadMode::Id => (self.id(at) >> ((at & 1) * 8)) as u8, + ReadMode::Cfi => { + if at & 1 == 0 { + Self::cfi(at / 4) + } else { + 0 + } + } + }; + value |= u64::from(data) << (byte * 8); + } + value + } + + fn write(&mut self, offset: u64, size: u8, value: u64) { + // A bank write is at most 32 bits. 64-bit array reads are supported, but + // 64-bit writes are not a single command/data cycle on the x32 bus. + if !Self::valid_access(offset, size) || size == 8 { + self.fail(SEQUENCE_ERROR); + return; + } + let offset = offset as usize; + let command = value as u8; + match self.state { + WriteState::Idle => self.command(offset, command), + WriteState::Program => { + // This cycle is data, even if its low byte resembles a command. + self.program(offset, size, value); + self.status |= READY; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + } + WriteState::Erase { block } => { + if command == 0xff { + self.read_array(); + } else if command != 0xd0 || Self::block(offset) != block { + self.fail(SEQUENCE_ERROR); + } else { + if self.locked[block] { + self.status |= ERASE_ERROR | BLOCK_LOCKED; + } else { + let start = block * BLOCK_SIZE as usize; + self.storage[start..start + BLOCK_SIZE as usize].fill(0xff); + } + self.status |= READY; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + } + } + WriteState::Lock { block } => { + if command == 0xff { + self.read_array(); + } else if !matches!(command, 0x01 | 0xd0) || Self::block(offset) != block { + self.fail(SEQUENCE_ERROR); + } else { + self.locked[block] = command == 0x01; + self.status |= READY; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + } + } + WriteState::BufferCount { block } => { + // Counts are per x16 chip and zero means one bank word. + let words = usize::from(value as u16) + 1; + if size != 4 + || offset & 3 != 0 + || Self::block(offset) != block + || words > BUFFER_WORDS + { + self.fail(SEQUENCE_ERROR); + } else { + self.state = WriteState::BufferData { + block, + remaining: words, + window: None, + }; + } + } + WriteState::BufferData { + block, + remaining, + window, + } => { + let data_window = offset & !(BUFFER_SIZE - 1); + if size != 4 + || offset & 3 != 0 + || Self::block(offset) != block + || window.is_some_and(|start| start != data_window) + { + self.fail(SEQUENCE_ERROR); + return; + } + self.buffer[self.buffered_words] = BufferedWord { + offset: offset as u32, + value: value as u32, + }; + self.buffered_words += 1; + self.state = if remaining == 1 { + WriteState::BufferConfirm + } else { + WriteState::BufferData { + block, + remaining: remaining - 1, + window: Some(data_window), + } + }; + } + WriteState::BufferConfirm => { + if command == 0xff { + self.read_array(); + } else if command != 0xd0 { + self.fail(SEQUENCE_ERROR); + } else { + // EDK2 confirms at DeviceBaseAddress, not the target block. + // Do not restrict the confirmation address to the data window. + for index in 0..self.buffered_words { + let word = self.buffer[index]; + self.program(word.offset as usize, 4, u64::from(word.value)); + } + self.buffered_words = 0; + self.status |= READY; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn command(flash: &mut NorFlash, offset: u64, command: u16) { + flash.write(offset, 4, u64::from(command) | (u64::from(command) << 16)); + } + + fn status(flash: &mut NorFlash) -> u64 { + command(flash, 0, 0x70); + flash.read(0, 4) + } + + fn program(flash: &mut NorFlash, offset: u64, value: u32) { + command(flash, offset, 0x40); + flash.write(offset, 4, u64::from(value)); + } + + #[test] + fn erased_array_and_programming_preserve_little_endian_accesses() { + let mut flash = NorFlash::new(); + assert_eq!(flash.read(0, 8), u64::MAX); + assert_eq!(flash.read(LEN - 1, 1), 0xff); + program(&mut flash, 0x100, 0x1234_5678); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0x100, 1), 0x78); + assert_eq!(flash.read(0x101, 2), 0x3456); + assert_eq!(flash.read(0x100, 4), 0x1234_5678); + assert_eq!(flash.read(0x100, 8), 0xffff_ffff_1234_5678); + assert_eq!(flash.read(0x103, 4), 0xffff_ff12); + + command(&mut flash, 0x104, 0x10); + flash.write(0x104, 2, 0x70ff); // Command-looking bytes are still data. + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0x104, 4), 0xffff_70ff); + command(&mut flash, 0x106, 0x40); + flash.write(0x106, 1, 0x90); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0x104, 4), 0xff90_70ff); + } + + #[test] + fn programming_cannot_set_bits_and_errors_are_sticky_until_cleared() { + let mut flash = NorFlash::new(); + program(&mut flash, 0, 0); + program(&mut flash, 0, 0xffff_ffff); + assert_eq!(status(&mut flash), 0x0090_0090); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0, 4), 0); + program(&mut flash, 4, 0x1234); + assert_eq!(status(&mut flash), 0x0090_0090); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(4, 4), 0x1234); + assert_eq!(status(&mut flash), 0x0080_0080); + } + + #[test] + fn erase_requires_confirmation_and_changes_only_the_selected_block() { + let mut flash = NorFlash::new(); + program(&mut flash, 0, 0); + program(&mut flash, BLOCK_SIZE, 0); + command(&mut flash, 0, 0x20); + assert_eq!(flash.storage[0], 0); + command(&mut flash, 0, 0xff); // Cancel before confirmation. + assert_eq!(flash.read(0, 4), 0); + command(&mut flash, 0x100, 0x20); + command(&mut flash, 0, 0xd0); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0, 4), 0xffff_ffff); + assert_eq!(flash.read(BLOCK_SIZE - 4, 4), 0xffff_ffff); + assert_eq!(flash.read(BLOCK_SIZE, 4), 0); + + command(&mut flash, BLOCK_SIZE, 0x20); + command(&mut flash, 0, 0xd0); // Wrong block must not erase anything. + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(BLOCK_SIZE, 4), 0); + } + + #[test] + fn ids_lock_unlock_and_cfi_describe_the_interleaved_bank() { + let mut flash = NorFlash::new(); + command(&mut flash, 0, 0x90); + assert_eq!(flash.read(0, 4), 0x0089_0089); + assert_eq!(flash.read(4, 4), 0x0018_0018); + assert_eq!(flash.read(0, 8), 0x0018_0018_0089_0089); + assert_eq!(flash.read(BLOCK_SIZE + 8, 4), 0); + + command(&mut flash, BLOCK_SIZE, 0x60); + command(&mut flash, BLOCK_SIZE, 0x01); + command(&mut flash, BLOCK_SIZE + 8, 0x90); + assert_eq!(flash.read(BLOCK_SIZE + 8, 4), 0x0001_0001); + program(&mut flash, BLOCK_SIZE, 0); + assert_eq!(status(&mut flash), 0x0092_0092); + command(&mut flash, 0, 0x50); + command(&mut flash, BLOCK_SIZE, 0x20); + command(&mut flash, BLOCK_SIZE, 0xd0); + assert_eq!(status(&mut flash), 0x00a2_00a2); + command(&mut flash, 0, 0x50); + command(&mut flash, BLOCK_SIZE, 0x60); + command(&mut flash, BLOCK_SIZE, 0xd0); + program(&mut flash, BLOCK_SIZE, 0); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(BLOCK_SIZE, 4), 0); + + command(&mut flash, 0x55 * 4, 0x98); + assert_eq!(flash.read(0x10 * 4, 4), 0x0051_0051); + assert_eq!(flash.read(0x11 * 4, 4), 0x0052_0052); + assert_eq!(flash.read(0x12 * 4, 4), 0x0059_0059); + assert_eq!(flash.read(0x27 * 4, 4), 0x0019_0019); + assert_eq!(flash.read(0x2a * 4, 4), 0x000b_000b); + assert_eq!(flash.read(0x2d * 4, 4), 0x00ff_00ff); + assert_eq!(flash.read(0x30 * 4, 4), 0x0002_0002); + assert_eq!(flash.read(0x52 * 4, 4), 0); + } + + #[test] + fn buffered_program_stages_data_until_base_address_confirmation() { + let mut flash = NorFlash::new(); + let target = BLOCK_SIZE + 128; + command(&mut flash, target, 0xe8); + assert_eq!(flash.read(target, 4), 0x0080_0080); + command(&mut flash, target, 1); // Two words. + flash.write(target, 4, 0x1234_5678); + flash.write(target + 4, 4, 0xd0ff_70e8); + assert_eq!(flash.storage[target as usize], 0xff); + command(&mut flash, 0, 0xd0); // EDK2 confirms at the bank base. + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(target, 8), 0xd0ff_70e8_1234_5678); + + command(&mut flash, target + 8, 0xe8); + command(&mut flash, target + 8, 0); // Count zero is one word. + flash.write(target + 8, 4, 0x1122_33ff); + command(&mut flash, 0, 0xd0); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(target + 8, 4), 0x1122_33ff); + + command(&mut flash, target, 0xe8); + command(&mut flash, target, 0); + flash.write(target, 4, 0xffff_ffff); + command(&mut flash, 0, 0xd0); + assert_eq!(status(&mut flash), 0x0090_0090); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(target, 4), 0x1234_5678); + } + + #[test] + fn full_edk2_buffer_and_last_bank_word_are_programmable() { + let mut flash = NorFlash::new(); + let target = LEN - 128; + command(&mut flash, target, 0xe8); + command(&mut flash, target, 31); + for index in 0..32 { + flash.write(target + index * 4, 4, 0xa500_0000 | index); + } + command(&mut flash, 0, 0xd0); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + for index in 0..32 { + assert_eq!(flash.read(target + index * 4, 4), 0xa500_0000 | index); + } + + let target = LEN - BUFFER_SIZE as u64; + command(&mut flash, target, 0xe8); + command(&mut flash, target, (BUFFER_WORDS - 1) as u16); + for index in 0..BUFFER_WORDS as u64 { + // Keep the earlier P30 data unchanged while testing the larger + // buffer capacity advertised by QEMU's CFI table. + let value = if index < BUFFER_WORDS as u64 - 32 { + 0x5a00_0000 | index + } else { + 0xa500_0000 | (index - (BUFFER_WORDS as u64 - 32)) + }; + flash.write(target + index * 4, 4, value); + } + command(&mut flash, 0, 0xd0); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(target, 4), 0x5a00_0000); + assert_eq!(flash.read(LEN - 4, 4), 0xa500_001f); + } + + #[test] + fn invalid_buffer_sequences_discard_staged_data() { + let mut flash = NorFlash::new(); + command(&mut flash, 0, 0xe8); + command(&mut flash, 0, BUFFER_WORDS as u16); // One word too many. + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + + command(&mut flash, 0, 0xe8); + command(&mut flash, 0, 1); + flash.write(0, 4, 0); + flash.write(BUFFER_SIZE as u64, 4, 0); // Cross the write-buffer window. + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(0, 8), u64::MAX); + assert_eq!(flash.read(BUFFER_SIZE as u64, 4), 0xffff_ffff); + + command(&mut flash, 0, 0xe8); + command(&mut flash, 0, 0); + flash.write(0, 4, 0); + command(&mut flash, 0, 0x70); // Not a confirmation. + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(0, 4), 0xffff_ffff); + + command(&mut flash, 0, 0xe8); + command(&mut flash, 0, 0); + flash.write(0, 4, 0); + command(&mut flash, 0, 0xff); // Explicit cancellation. + assert_eq!(flash.read(0, 4), 0xffff_ffff); + } + + #[test] + fn invalid_commands_and_accesses_are_bounded_and_recoverable() { + let mut flash = NorFlash::new(); + assert_eq!(flash.read(LEN, 1), 0); + assert_eq!(flash.read(LEN - 3, 4), 0); + assert_eq!(flash.read(u64::MAX, 8), 0); + assert_eq!(flash.read(0, 0), 0); + assert_eq!(flash.read(0, 3), 0); + flash.write(LEN - 3, 4, 0); + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(LEN - 4, 4), 0xffff_ffff); + command(&mut flash, 0, 0xab); + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + program(&mut flash, LEN - 4, 0x1234_5678); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(LEN - 4, 4), 0x1234_5678); + + command(&mut flash, 0, 0x40); + flash.write(u64::MAX, 4, 0); + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(0, 4), 0xffff_ffff); + flash.write(0, 8, 0); + assert_eq!(status(&mut flash), 0x00b0_00b0); + } +} diff --git a/src/emulator/src/devices/pl031.rs b/src/emulator/src/devices/pl031.rs new file mode 100644 index 0000000..3f48942 --- /dev/null +++ b/src/emulator/src/devices/pl031.rs @@ -0,0 +1,286 @@ +//! QEMU virt's PL031 real-time clock, driven by the host monotonic clock. +//! +//! Like QEMU's PL031, the counter is permanently enabled. Alarm status is +//! latched when the device is accessed; no interrupt line is connected. + +use core::time::Duration; +use std::time::Instant; + +use super::bus::Device; + +pub const BASE: u64 = 0x0901_0000; +pub const LEN: u64 = 0x1000; + +const DR: u64 = 0x00; +const MR: u64 = 0x04; +const LR: u64 = 0x08; +const CR: u64 = 0x0c; +const IMSC: u64 = 0x10; +const RIS: u64 = 0x14; +const MIS: u64 = 0x18; +const ICR: u64 = 0x1c; +const PERIPH_ID0: u64 = 0xfe0; +const IDENTIFICATION: [u32; 8] = [0x31, 0x10, 0x14, 0x00, 0x0d, 0xf0, 0x05, 0xb1]; + +/// A wrapping 32-bit seconds counter with PL031 alarm and identification registers. +pub struct Pl031 { + origin: Instant, + tick_offset: u32, + load: u32, + match_value: u32, + interrupt_mask: u32, + raw_interrupt: u32, + alarm_deadline: Option, +} + +impl Pl031 { + /// Start at `epoch_seconds`; subsequent time is elapsed host monotonic time. + pub fn new(epoch_seconds: u32) -> Self { + Self::new_at(epoch_seconds, Instant::now()) + } + + fn new_at(epoch_seconds: u32, now: Instant) -> Self { + Self { + origin: now, + tick_offset: epoch_seconds, + load: 0, + match_value: 0, + interrupt_mask: 0, + raw_interrupt: 0, + alarm_deadline: None, + } + } + + fn count(&self, elapsed: Duration) -> u32 { + self.tick_offset.wrapping_add(elapsed.as_secs() as u32) + } + + fn refresh_alarm(&mut self, elapsed: Duration) { + if self + .alarm_deadline + .is_some_and(|deadline| elapsed >= deadline) + { + self.raw_interrupt = 1; + self.alarm_deadline = None; + } + } + + fn set_alarm(&mut self, elapsed: Duration) { + // Unsigned subtraction schedules a past match after the counter wraps. + // QEMU raises an already matching alarm immediately, otherwise schedules + // it this many seconds after programming MR or LR. + let ticks = self.match_value.wrapping_sub(self.count(elapsed)); + if ticks == 0 { + self.raw_interrupt = 1; + self.alarm_deadline = None; + } else { + self.alarm_deadline = Some(elapsed + Duration::from_secs(u64::from(ticks))); + } + } + + fn access(offset: u64, size: u8) -> Option<(u64, u32, u32)> { + // The APB register width is 32 bits. Byte and halfword accesses select + // little-endian lanes, including the byte-sized PrimeCell IDs. + if !matches!(size, 1 | 2 | 4) || offset >= LEN || (offset & 3) + u64::from(size) > 4 { + return None; + } + let shift = ((offset & 3) * 8) as u32; + let mask = u32::MAX >> (32 - u32::from(size) * 8); + Some((offset & !3, shift, mask)) + } + + fn read_at(&mut self, offset: u64, size: u8, now: Instant) -> u64 { + let Some((register, shift, mask)) = Self::access(offset, size) else { + return 0; + }; + let elapsed = now.duration_since(self.origin); + self.refresh_alarm(elapsed); + let value = match register { + DR => self.count(elapsed), + MR => self.match_value, + LR => self.load, + CR => 1, // QEMU's RTC is permanently enabled. + IMSC => self.interrupt_mask, + RIS => self.raw_interrupt, + MIS => self.raw_interrupt & self.interrupt_mask, + PERIPH_ID0..=0xffc => IDENTIFICATION[((register - PERIPH_ID0) / 4) as usize], + _ => 0, // Includes the write-only interrupt clear register. + }; + u64::from((value >> shift) & mask) + } + + fn write_at(&mut self, offset: u64, size: u8, value: u64, now: Instant) { + let Some((register, shift, mask)) = Self::access(offset, size) else { + return; + }; + let elapsed = now.duration_since(self.origin); + self.refresh_alarm(elapsed); + let bits = ((value as u32) & mask) << shift; + let lane_mask = mask << shift; + match register { + MR => { + self.match_value = (self.match_value & !lane_mask) | bits; + self.set_alarm(elapsed); + } + LR => { + self.load = (self.load & !lane_mask) | bits; + // Keep the oscillator's subsecond phase when setting the time. + self.tick_offset = self + .tick_offset + .wrapping_add(self.load.wrapping_sub(self.count(elapsed))); + self.set_alarm(elapsed); + } + IMSC => self.interrupt_mask = ((self.interrupt_mask & !lane_mask) | bits) & 1, + ICR => self.raw_interrupt &= !bits, + // CR writes are ignored, as in QEMU; all remaining registers are + // read-only or reserved. + _ => {} + } + } +} + +impl Device for Pl031 { + fn read(&mut self, offset: u64, size: u8) -> u64 { + self.read_at(offset, size, Instant::now()) + } + + fn write(&mut self, offset: u64, size: u8, value: u64) { + self.write_at(offset, size, value, Instant::now()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counter_advances_in_whole_seconds_and_wraps() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(u32::MAX - 1, start); + assert_eq!(rtc.read_at(DR, 4, start), u64::from(u32::MAX - 1)); + assert_eq!( + rtc.read_at(DR, 4, start + Duration::from_millis(999)), + u64::from(u32::MAX - 1) + ); + assert_eq!( + rtc.read_at(DR, 4, start + Duration::from_secs(1)), + u64::from(u32::MAX) + ); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(2)), 0); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(3)), 1); + } + + #[test] + fn loading_time_preserves_fractional_phase_and_load_readback() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(100, start); + rtc.write_at(LR, 4, 500, start + Duration::from_millis(1750)); + assert_eq!(rtc.read_at(LR, 4, start + Duration::from_millis(1750)), 500); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_millis(1999)), 500); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(2)), 501); + assert_eq!(rtc.read_at(LR, 4, start + Duration::from_secs(2)), 500); + rtc.write_at(LR, 4, 20, start + Duration::from_secs(3)); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(3)), 20); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(4)), 21); + } + + #[test] + fn control_is_permanently_enabled_like_qemu() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(100, start); + assert_eq!(rtc.read_at(CR, 4, start), 1); + rtc.write_at(CR, 4, 0, start); + assert_eq!(rtc.read_at(CR, 4, start), 1); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(2)), 102); + rtc.write_at(CR, 4, u64::MAX, start + Duration::from_secs(2)); + assert_eq!(rtc.read_at(CR, 4, start + Duration::from_secs(2)), 1); + } + + #[test] + fn match_latches_raw_status_independently_of_the_mask() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(100, start); + rtc.write_at(MR, 4, 102, start); + assert_eq!(rtc.read_at(MR, 4, start), 102); + assert_eq!(rtc.read_at(RIS, 4, start + Duration::from_secs(1)), 0); + let after = start + Duration::from_secs(5); + // A missed matching second still latches the interrupt. + assert_eq!(rtc.read_at(RIS, 4, after), 1); + assert_eq!(rtc.read_at(MIS, 4, after), 0); + rtc.write_at(IMSC, 4, 0xffff, after); + assert_eq!(rtc.read_at(IMSC, 4, after), 1); + assert_eq!(rtc.read_at(MIS, 4, after), 1); + rtc.write_at(IMSC, 4, 0, after); + assert_eq!(rtc.read_at(MIS, 4, after), 0); + assert_eq!(rtc.read_at(RIS, 4, after), 1); + rtc.write_at(ICR, 4, 2, after); + assert_eq!(rtc.read_at(RIS, 4, after), 1); + rtc.write_at(ICR, 4, 1, after); + assert_eq!(rtc.read_at(RIS, 4, after), 0); + assert_eq!(rtc.read_at(ICR, 4, after), 0); + } + + #[test] + fn match_scheduling_wraps_and_load_reschedules_the_alarm() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(u32::MAX - 1, start); + rtc.write_at(MR, 4, 1, start); + assert_eq!(rtc.read_at(RIS, 4, start + Duration::from_secs(2)), 0); + assert_eq!(rtc.read_at(RIS, 4, start + Duration::from_secs(3)), 1); + let now = start + Duration::from_secs(3); + rtc.write_at(ICR, 4, 1, now); + assert_eq!(rtc.read_at(RIS, 4, now), 0); + rtc.write_at(MR, 4, 1, now); // Equal to the current count. + assert_eq!(rtc.read_at(RIS, 4, now), 1); + rtc.write_at(ICR, 4, 1, now); + rtc.write_at(MR, 4, 10, now); + rtc.write_at(LR, 4, 9, now); + assert_eq!(rtc.read_at(RIS, 4, now), 0); + assert_eq!(rtc.read_at(RIS, 4, now + Duration::from_secs(1)), 1); + rtc.write_at(ICR, 4, 1, now + Duration::from_secs(1)); + rtc.write_at(LR, 4, 10, now + Duration::from_secs(1)); + assert_eq!(rtc.read_at(RIS, 4, now + Duration::from_secs(1)), 1); + } + + #[test] + fn primecell_ids_and_little_endian_register_lanes_are_readable() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(0x1234_5678, start); + for (index, expected) in IDENTIFICATION.into_iter().enumerate() { + let offset = PERIPH_ID0 + index as u64 * 4; + assert_eq!(rtc.read_at(offset, 1, start), u64::from(expected)); + assert_eq!(rtc.read_at(offset, 4, start), u64::from(expected)); + assert_eq!(rtc.read_at(offset + 1, 1, start), 0); + } + assert_eq!(rtc.read_at(DR, 1, start), 0x78); + assert_eq!(rtc.read_at(DR + 1, 1, start), 0x56); + assert_eq!(rtc.read_at(DR + 2, 2, start), 0x1234); + rtc.write_at(MR, 4, 0x1122_3344, start); + rtc.write_at(MR + 1, 1, 0xaa, start); + assert_eq!(rtc.read_at(MR, 4, start), 0x1122_aa44); + rtc.write_at(LR, 4, 0x0102_0304, start); + rtc.write_at(LR + 2, 2, 0xaabb, start); + assert_eq!(rtc.read_at(LR, 4, start), 0xaabb_0304); + assert_eq!(rtc.read_at(DR, 4, start), 0xaabb_0304); + } + + #[test] + fn read_only_reserved_and_invalid_accesses_do_not_change_registers() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(100, start); + for offset in [DR, RIS, MIS, PERIPH_ID0, 0x20, LEN, u64::MAX] { + rtc.write_at(offset, 4, u64::MAX, start); + } + assert_eq!(rtc.read_at(DR, 4, start), 100); + assert_eq!(rtc.read_at(RIS, 4, start), 0); + assert_eq!(rtc.read_at(MIS, 4, start), 0); + assert_eq!(rtc.read_at(PERIPH_ID0, 1, start), 0x31); + for (offset, size) in [(MR, 0), (MR, 3), (MR, 8), (MR + 3, 2), (LEN, 4)] { + rtc.write_at(offset, size, u64::MAX, start); + assert_eq!(rtc.read_at(offset, size, start), 0); + } + assert_eq!(rtc.read_at(MR, 4, start), 0); + assert_eq!(rtc.read_at(0x20, 4, start), 0); + assert_eq!(rtc.read_at(LEN, 4, start), 0); + } +} diff --git a/src/emulator/src/system/boot.rs b/src/emulator/src/system/boot.rs index 64faa61..82a890d 100644 --- a/src/emulator/src/system/boot.rs +++ b/src/emulator/src/system/boot.rs @@ -208,7 +208,7 @@ pub fn prepare_firmware( ram_base: u64, ram_size: u64, ) -> Result { - let tree = super::fdt::build(ram_base, ram_size, ""); + let tree = super::fdt::build_firmware(ram_base, ram_size); let end = ram_base.checked_add(ram_size).ok_or(Error::NoRoom)?; if ram_base .checked_add(tree.len() as u64) diff --git a/src/emulator/src/system/fdt.rs b/src/emulator/src/system/fdt.rs index 6efe45f..cb573fe 100644 --- a/src/emulator/src/system/fdt.rs +++ b/src/emulator/src/system/fdt.rs @@ -110,6 +110,11 @@ pub fn build(ram_base: u64, ram_size: u64, cmdline: &str) -> Vec { build_smp(ram_base, ram_size, cmdline, 1, None, false) } +/// Firmware discovers fw_cfg in the tree; direct kernel boots do not expose it. +pub fn build_firmware(ram_base: u64, ram_size: u64) -> Vec { + build_tree(ram_base, ram_size, "", 1, None, false, true) +} + pub fn build_smp( ram_base: u64, ram_size: u64, @@ -117,6 +122,18 @@ pub fn build_smp( cpus: u32, initrd: Option<(u64, u64)>, has_virtio: bool, +) -> Vec { + build_tree(ram_base, ram_size, cmdline, cpus, initrd, has_virtio, false) +} + +fn build_tree( + ram_base: u64, + ram_size: u64, + cmdline: &str, + cpus: u32, + initrd: Option<(u64, u64)>, + has_virtio: bool, + firmware: bool, ) -> Vec { let mut b = Builder::new(); b.begin(""); @@ -166,9 +183,22 @@ pub fn build_smp( b.cells("#address-cells", &[2]); b.cells("#size-cells", &[2]); b.property("interrupt-controller", &[]); - b.string("compatible", "arm,cortex-a15-gic"); - let mut registers = Vec::from(range(GICD_BASE, 0x1000)); - registers.extend_from_slice(&range(GIC_CPU_BASE, 0x2000)); + let registers = if firmware { + b.string("compatible", "arm,gic-v3"); + b.cells("#redistributor-regions", &[1]); + b.property("ranges", &[]); + let mut registers = Vec::from(range(GICD_BASE, crate::devices::gicv3::DISTRIBUTOR_SIZE)); + registers.extend_from_slice(&range( + super::afdt::GICR_BASE, + crate::devices::gicv3::REDISTRIBUTOR_SIZE, + )); + registers + } else { + b.string("compatible", "arm,cortex-a15-gic"); + let mut registers = Vec::from(range(GICD_BASE, 0x1000)); + registers.extend_from_slice(&range(GIC_CPU_BASE, 0x2000)); + registers + }; b.cells("reg", ®isters); b.cells("phandle", &[2]); b.end(); @@ -179,6 +209,57 @@ pub fn build_smp( b.cells("clocks", &[1, 1]); b.string("clock-names", "uartclk\0apb_pclk"); b.end(); + if firmware { + b.begin("fw-cfg@9020000"); + b.string("compatible", "qemu,fw-cfg-mmio"); + b.cells( + "reg", + &range(crate::devices::fw_cfg::BASE, crate::devices::fw_cfg::LEN), + ); + b.end(); + b.begin("pl031@9010000"); + b.string("compatible", "arm,pl031\0arm,primecell"); + b.cells( + "reg", + &range(crate::devices::pl031::BASE, crate::devices::pl031::LEN), + ); + b.cells("interrupts", &[0, 2, 4]); + b.cells("clocks", &[1]); + b.string("clock-names", "apb_pclk"); + b.end(); + b.begin("flash@0"); + b.string("compatible", "cfi-flash"); + b.cells("bank-width", &[4]); + let mut registers = Vec::from(range(0, crate::devices::nor_flash::LEN)); + registers.extend_from_slice(&range( + crate::devices::nor_flash::BASE, + crate::devices::nor_flash::LEN, + )); + b.cells("reg", ®isters); + b.end(); + b.begin("pcie@10000000"); + b.string("compatible", "pci-host-ecam-generic"); + b.string("device_type", "pci"); + b.cells("#address-cells", &[3]); + b.cells("#size-cells", &[2]); + b.cells("#interrupt-cells", &[1]); + b.cells("bus-range", &[0, 255]); + b.cells("linux,pci-domain", &[0]); + b.cells( + "reg", + &range( + crate::devices::pci::ECAM_BASE, + crate::devices::pci::ECAM_SIZE, + ), + ); + // Memory BAR space, matching QEMU virt. This host has no port-I/O devices. + b.cells( + "ranges", + &[0x0200_0000, 0, 0x1000_0000, 0, 0x1000_0000, 0, 0x2eff_0000], + ); + b.property("dma-coherent", &[]); + b.end(); + } if has_virtio { b.begin("virtio@a000000"); b.string("compatible", "virtio,mmio"); @@ -225,4 +306,33 @@ mod tests { assert!(tree.windows(17).any(|part| part == b"linux,initrd-end\0")); assert!(tree.windows(12).any(|part| part == b"virtio,mmio\0")); } + + #[test] + fn firmware_tree_describes_only_firmware_devices_and_gicv3() { + let tree = build_firmware(0x4000_0000, 4 << 30); + for compatible in [ + b"arm,gic-v3\0".as_slice(), + b"qemu,fw-cfg-mmio\0", + b"arm,pl031\0", + b"cfi-flash\0", + b"pci-host-ecam-generic\0", + ] { + assert!( + tree.windows(compatible.len()) + .any(|part| part == compatible) + ); + } + assert!(!tree.windows(17).any(|part| part == b"arm,cortex-a15-gic\0")); + assert!(!tree.windows(12).any(|part| part == b"virtio,mmio\0")); + assert!( + tree.windows(4) + .any(|part| part == (crate::devices::fw_cfg::BASE as u32).to_be_bytes()) + ); + let kernel_tree = build(0x4000_0000, 128 << 20, ""); + assert!( + !kernel_tree + .windows(16) + .any(|part| part == b"qemu,fw-cfg-mmio\0") + ); + } } diff --git a/src/emulator/src/system/machine.rs b/src/emulator/src/system/machine.rs index 146ea6b..49ff311 100644 --- a/src/emulator/src/system/machine.rs +++ b/src/emulator/src/system/machine.rs @@ -44,15 +44,14 @@ pub trait DeviceIo { fn write_virtio(&mut self, offset: u64, size: u8, value: u64) { let _ = (offset, size, value); } - /// A device the board places anywhere in the address space (the M1 SoC's - /// interrupt controller and UART). `None` declines the access, which then goes - /// to the fixed windows below. - fn read_soc(&mut self, cpu_id: u32, address: u64, size: u8) -> Option { + /// A board-specific device (SoC peripherals or firmware devices). `None` + /// declines the access, which then goes to PCI and the fixed windows below. + fn read_board(&mut self, cpu_id: u32, address: u64, size: u8) -> Option { let _ = (cpu_id, address, size); None } /// Returns `false` when the board's devices do not claim the address. - fn write_soc(&mut self, cpu_id: u32, address: u64, size: u8, value: u64) -> bool { + fn write_board(&mut self, cpu_id: u32, address: u64, size: u8, value: u64) -> bool { let _ = (cpu_id, address, size, value); false } @@ -1189,20 +1188,20 @@ impl Machine { let window = |base, length| { fixed && physical >= base && end.is_some_and(|end| end <= base + length) }; - let soc = if load { - devices.read_soc(self.cpu_id, physical, width).map(Some) + let board = if load { + devices.read_board(self.cpu_id, physical, width).map(Some) } else { devices - .write_soc(self.cpu_id, physical, width, value) + .write_board(self.cpu_id, physical, width, value) .then_some(None) }; // Only accesses the board's own devices decline reach the PCI bus. - let pci = match (&soc, load) { + let pci = match (&board, load) { (Some(_), _) => None, (None, true) => devices.read_pci(physical, width).map(Some), (None, false) => devices.write_pci(physical, width, value).then_some(None), }; - let read = if let Some(read) = soc { + let read = if let Some(read) = board { read } else if let Some(read) = pci { read diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index e13845a..d216a5e 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -284,6 +284,32 @@ impl Intc { } } +/// Peripherals the firmware discovers in its device tree, separate from kernel boots. +fn firmware_devices(ram_size: u64) -> crate::devices::Bus { + use crate::devices::{Bus, BusDevice, fw_cfg, nor_flash, pl031}; + let mut bus = Bus::new(); + bus.attach(BusDevice::new( + fw_cfg::BASE, + fw_cfg::LEN, + fw_cfg::FwCfg::new(ram_size), + )); + bus.attach(BusDevice::new( + nor_flash::BASE, + nor_flash::LEN, + nor_flash::NorFlash::new(), + )); + let seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as u32; + bus.attach(BusDevice::new( + pl031::BASE, + pl031::LEN, + pl031::Pl031::new(seconds), + )); + bus +} + /// The bus of one board. The fixed windows (QEMU `virt`'s UART, GIC and virtio) exist /// only on the virt board; on the M1 board `soc` answers instead, and the GIC accessors /// below are never reached with [`Intc::Apple`]. @@ -293,6 +319,7 @@ struct SystemDevices<'a, V> { virtio: Option<&'a mut V>, soc: Option<&'a mut m1::Soc>, pci: Option<&'a mut PciHost>, + firmware: Option<&'a mut crate::devices::Bus>, } impl machine::DeviceIo for SystemDevices<'_, V> { @@ -359,13 +386,21 @@ impl machine::DeviceIo for SystemDevices<'_, V> { v.write(offset, size, value); } } - fn read_soc(&mut self, cpu_id: u32, address: u64, size: u8) -> Option { - self.soc.as_mut()?.read(cpu_id, address, size) + fn read_board(&mut self, cpu_id: u32, address: u64, size: u8) -> Option { + if let Some(soc) = self.soc.as_mut() { + soc.read(cpu_id, address, size) + } else { + self.firmware.as_mut()?.read(address, size) + } } - fn write_soc(&mut self, cpu_id: u32, address: u64, size: u8, value: u64) -> bool { - self.soc - .as_mut() - .is_some_and(|soc| soc.write(cpu_id, address, size, value)) + fn write_board(&mut self, cpu_id: u32, address: u64, size: u8, value: u64) -> bool { + if let Some(soc) = self.soc.as_mut() { + soc.write(cpu_id, address, size, value) + } else { + self.firmware + .as_mut() + .is_some_and(|bus| bus.write(address, size, value)) + } } fn read_pci(&mut self, address: u64, size: u8) -> Option { self.pci.as_mut()?.read(address, size) @@ -596,6 +631,8 @@ fn boot_machine( ), (Payload::Firmware(_), disk) => (None, Some(firmware_pci(disk, shared_mem.clone()))), }; + let mut firmware = + matches!(payload, Payload::Firmware(_)).then(|| firmware_devices(ram_size as u64)); let mut machine = machine::Machine::new(shared_mem); if apple { m1::identify(&mut machine.cpu); @@ -648,6 +685,7 @@ fn boot_machine( virtio: virtio.as_mut(), soc: soc.as_mut(), pci: pci.as_mut(), + firmware: firmware.as_mut(), }; match machine.run_with_devices(&mut dev) { Ok(machine::Exit::Timer) => {