diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93135c9..f355096 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest - name: Setup Mold Linker - uses: rui314/setup-mold@v1 + uses: rui314/setup-mold@staging - name: Run tests with nextest run: mold -run cargo nextest run --no-fail-fast --workspace --all-features lint: @@ -35,7 +35,7 @@ jobs: - name: Cache Cargo dependencies uses: Swatinem/rust-cache@v2 - name: Setup Mold Linker - uses: rui314/setup-mold@v1 + uses: rui314/setup-mold@staging - name: cargo fmt run: mold -run cargo fmt --all --check - name: Run lint diff --git a/Cargo.lock b/Cargo.lock index 7145c02..c74df12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,150 +2,18 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - -[[package]] -name = "awint" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a8f298efd080b0d3e1ee1723b5d88ecc0d78805fad347e09b76dbd4dadbe37" -dependencies = [ - "awint_core", - "awint_dag", - "awint_ext", - "awint_macro_internals", - "awint_macros", -] - -[[package]] -name = "awint_core" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24d226d9faa090abec192c8ad1ea74635e6cc94000c72a95ceb60cf691428c3" -dependencies = [ - "awint_internals", - "const_fn", -] - -[[package]] -name = "awint_dag" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5377016f3077b18304da4233a28626ea069047ca84c9bc8c8b81b659f11032" -dependencies = [ - "awint_ext", - "awint_macro_internals", - "awint_macros", - "smallvec", -] - -[[package]] -name = "awint_ext" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ff843cdefdd48fbe9bc11292a902b7921ff37433ed4124fa02136f83186f58" -dependencies = [ - "awint_core", - "const_fn", -] - -[[package]] -name = "awint_internals" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6209d7873685ac95ee41c4b63fe29bff9b2e1106de65d05b106b2b769c178575" -dependencies = [ - "const_fn", -] - -[[package]] -name = "awint_macro_internals" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b519a9cd37bf73e44e29b3779ca90246412b56db4af4e96d959f5082bb8bdcc" -dependencies = [ - "awint_ext", - "proc-macro2", - "triple_arena", -] - -[[package]] -name = "awint_macros" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "518a133c2163e8f021f708d7278af27dd6c1bcee96ea672a2d825b13a6931818" -dependencies = [ - "awint_macro_internals", -] - [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -[[package]] -name = "bytemuck" -version = "1.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - [[package]] name = "cfg-if" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" -[[package]] -name = "combine" -version = "4.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" -dependencies = [ - "bytes", - "memchr", -] - -[[package]] -name = "const_fn" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413d67b29ef1021b4d60f4aa1e925ca031751e213832b4b1d588fae623c05c60" - -[[package]] -name = "convert_case" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "crc32fast" version = "1.5.2" @@ -155,18 +23,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "downcast-rs" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" - -[[package]] -name = "dyn-clone" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" - [[package]] name = "equivalent" version = "1.0.2" @@ -191,20 +47,9 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" dependencies = [ - "allocator-api2", - "equivalent", "foldhash", ] -[[package]] -name = "hi_sparse_bitset" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "682097abdf22a902b7b2b5f2824e773691507289873c3ea0abbc5d4c85f05ec0" -dependencies = [ - "wide", -] - [[package]] name = "indexmap" version = "2.14.2" @@ -215,41 +60,12 @@ dependencies = [ "hashbrown", ] -[[package]] -name = "inventory" -version = "0.3.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" -dependencies = [ - "rustversion", -] - [[package]] name = "libc" version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" -[[package]] -name = "linkme" -version = "0.3.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" -dependencies = [ - "linkme-impl", -] - -[[package]] -name = "linkme-impl" -version = "0.3.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - [[package]] name = "lock_api" version = "0.4.14" @@ -259,12 +75,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - [[package]] name = "memchr" version = "2.8.3" @@ -283,12 +93,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "once_vec" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b61fbf7ec37342ce171d9f5e3b82be74bda6c4e84e3b28338a03d553913e6aa" - [[package]] name = "parking_lot" version = "0.12.5" @@ -312,49 +116,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "pliron" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d59d619aada2eb8f68bacac30cfa373f4e247c7667cdac69370f5eaf0b87178" -dependencies = [ - "awint", - "combine", - "downcast-rs", - "dyn-clone", - "hashbrown", - "hi_sparse_bitset", - "indexmap", - "inventory", - "linkme", - "log", - "once_vec", - "pliron-derive", - "regex", - "rustc-hash", - "rustc_apfloat", - "slotmap", - "smallvec", - "spin", - "thiserror", - "utf8-chars", -] - -[[package]] -name = "pliron-derive" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e9d7f08f5162e44988420d46a12bd9aa776a9c5a0bee5d1555741060c87a73" -dependencies = [ - "combine", - "convert_case", - "indexmap", - "proc-macro2", - "quote", - "rustc-hash", - "syn 3.0.6", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -373,12 +134,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "recasting" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3f51d3a1399f13fa081d97345d0fdf08597fe78558fc8539d0e67412f83b26" - [[package]] name = "redox_syscall" version = "0.5.18" @@ -388,96 +143,18 @@ dependencies = [ "bitflags", ] -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustc_apfloat" -version = "0.2.3+llvm-462a31f5a5ab" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" -dependencies = [ - "bitflags", - "smallvec", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "safe_arch" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42c6efa15875e6ecb39ca61fb0b0c1a40b84fac5a5ffe71eef7d1000c8eb3f5f" -dependencies = [ - "bytemuck", -] - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "slotmap" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038" -dependencies = [ - "version_check", -] - [[package]] name = "smallvec" version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" -[[package]] -name = "spin" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0134f9043ed38b087ac4f7d4af44c79e2c9e5094421fe3164f435ce585953b10" -dependencies = [ - "lock_api", -] - [[package]] name = "syn" version = "2.0.119" @@ -489,46 +166,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "thiserror" -version = "2.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "triple_arena" -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee55b64228381514ddc4ad004e631ccf45e594e428d13ce197697ce56ef1c55" -dependencies = [ - "recasting", -] - [[package]] name = "unicode-ident" version = "1.0.26" @@ -536,26 +173,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] -name = "unicode-segmentation" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" - -[[package]] -name = "utf8-chars" -version = "3.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92498c67ea3511b37eccff13b573ed871faf0ce9c4056ab032bd01a681f445a0" +name = "volt-disasm" +version = "0.1.0" dependencies = [ - "arrayvec", + "object", + "volt-isa-aarch64", ] -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "volt-emulator" version = "0.1.0" @@ -563,6 +187,8 @@ dependencies = [ "libc", "parking_lot", "volt-ir", + "volt-isa-aarch64", + "volt-lift-aarch64", "volt-target", ] @@ -578,8 +204,18 @@ dependencies = [ [[package]] name = "volt-ir" version = "0.1.0" + +[[package]] +name = "volt-isa-aarch64" +version = "0.1.0" + +[[package]] +name = "volt-lift-aarch64" +version = "0.1.0" dependencies = [ - "pliron", + "volt-ir", + "volt-isa-aarch64", + "volt-target", ] [[package]] @@ -589,16 +225,7 @@ dependencies = [ "libc", "object", "volt-ir", -] - -[[package]] -name = "wide" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d920ac99c3c8edce110cb8d07dbb324d6d026011dce85b1e9355b70f0adacc4f" -dependencies = [ - "bytemuck", - "safe_arch", + "volt-isa-aarch64", ] [[package]] @@ -634,7 +261,7 @@ checksum = "09359adf85eecc7ca2696928137722055b85d380089b159673a161347115adc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fb282c3..e369bd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,9 +2,12 @@ resolver = "3" members = [ "src/target", + "src/isa/aarch64", + "src/disasm", "src/ir", "src/emulator", "src/emulator/cli", + "src/lift/aarch64", ] [workspace.package] @@ -14,8 +17,10 @@ license = "Apache-2.0" [workspace.dependencies] volt-ir = { path = "src/ir" } +volt-isa-aarch64 = { path = "src/isa/aarch64" } volt-target = { path = "src/target" } volt-emulator = { path = "src/emulator" } +volt-lift-aarch64 = { path = "src/lift/aarch64" } object = { version = "0.40.0", default-features = false, features = ["read_core", "write_core", "elf", "macho"] } libc = "0.2" diff --git a/README.md b/README.md index 7bab141..4d3d3f6 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,35 @@ Independent Rust workspace for modular dynamic binary translation. +## Crates + +| Crate | Path | Role | +|---|---|---| +| `volt-isa-aarch64` | `src/isa/aarch64` | The AArch64 instruction set and nothing else: the single decoder (`decode`, delegating to the `fp`, `simd`, `pauth` and `gxf` families), control-flow classification (`flow`) and assembly text (`format`, `objdump` spelling). `no_std`; knows nothing of guest state, memory, blocks or executable memory. | +| `volt-ir` | `src/ir` | Typed SSA/CFG intermediate representation, verifier, legalization, bitcode. | +| `volt-target` | `src/target` | Native code generation (AArch64, x86-64), register allocation, object/link output, executable memory. | +| `volt-lift-aarch64` | `src/lift/aarch64` | AArch64 instruction semantics lifted to `volt-ir`, generic over an `Environment`. | +| `volt-emulator` | `src/emulator` | Machine models, translation-block policy (`aarch64::host::ends_block`), host execution of the instruction families the lifter does not implement, devices. | +| `volt-disasm` | `src/disasm` | Binary images, linear listing, control-flow recovery. | + +Every consumer of an instruction word decodes it once through `volt_isa_aarch64::decode` +and works from the result; nothing else in the workspace masks instruction bits. Where a +translator cuts its blocks is that translator's policy, not a property of an instruction. + +`cargo run --release -p volt-disasm --example fmtdiff -- [mnemonic] [limit]` checks +the formatter against GNU `objdump`, and `--example coverage -- ` reports which words +the decoder does not recognise. + +## AArch64 lifter + +`volt-lift-aarch64` (`src/lift/aarch64`) is a `no_std` crate that lifts decoded AArch64 +instructions to `volt-ir` over guest state held in memory: each lifted function is +`fn(state: *mut u8) -> u64` and returns the next guest pc. It implements the architecture +(integer ALU, flags, branches, register-only SIMD) and leaves the machine to an +`Environment` trait: the state layout, how memory is reached, and system instructions. +`volt-emulator` is one environment (`Cpu` state, an inline data TLB, trap-and-resume slow +paths); a user-mode emulator or a static translator supplies its own. + ## AArch64 Linux boot `volt-boot` launches a raw arm64 Linux `Image` with a PL011 console, GICv2, @@ -32,6 +61,14 @@ The disk form supplies `root=/dev/vda rw` by default. `--cpus`, `--idle`, `--fast`, and `--spin` control the virtual CPU count and WFI policy. Use `volt-boot --help` for the complete generated interface. +Firmware boots use QEMU `virt` firmware devices: GICv3, fw_cfg, PL031 RTC, PCI ECAM and +two CFI NOR banks. The second flash bank stores volatile UEFI variables; no state persists +between runs. + +```sh +cargo run --release --bin volt-boot -- --bios /path/to/QEMU_EFI.fd --timeout 20 +``` + ## XNU boot `volt-boot` also boots an XNU kernel. A Mach-O image is detected by its magic, so no @@ -67,6 +104,79 @@ enforced by translation, the physical timer is stored but never fires, floating rounds to nearest even and sets no `FPSR` flags, and the GICv3 carries only the virtual timer (no SPIs, SGIs or group 1). +## Apple M1 (t8103) SoC + +`--dtb` selects the M1 machine in place of the QEMU `virt` board. The device tree is +the Asahi kernel's board tree, which the kernel tree builds as +`arch/arm64/boot/dts/apple/t8103-j274.dtb`: + +```sh +cargo run --release --bin volt-boot -- \ + ~/src/linux/arch/arm64/boot/Image 60 \ + --dtb ~/src/linux/arch/arm64/boot/dts/apple/t8103-j274.dtb +``` + +The SoC addresses come from the M1 Mac mini's own device tree (`DeviceTree.j274ap` +in a macOS restore image; `ipsw dtree` reads it). DRAM is at `0x8_0000_0000`, the +interrupt controller (AIC) at `0x2_3b10_0000`, the power-state blocks at +`0x2_3b70_0000` and `0x2_3d28_0000`, and the console (`apple,s5l-uart`, `stdout-path`) +at `0x2_3520_0000`. The machine is one vCPU with the M1 Icestorm identity +(`MIDR_EL1` 0x610f0220) and a 16 KiB granule, which the Asahi kernel requires. + +The Linux kernel boots through the AIC and the S5L console, probes its power domains, +and panics on the missing root filesystem (`VFS: Unable to mount root fs`), which is +the expected result without one. `--initrd`, `--disk` and `--smp` above 1 are rejected +on this board. A kernelcache from the restore image (`MH_FILESET`) loads, with its ADT +as `--dtb`, and enters at its reset trampoline with the cold-boot reset type. It runs +several hundred blocks of early boot, then executes `genter` (guarded execution) before +it has set `VBAR_EL1`. The kernel expects iBoot to have enabled GXF and set its monitor +entry, which volt does not provide; chained fixups are still not applied. + +The EL2 regime is modelled: `CurrentEl` reports EL2, exceptions and `ERET` target +`VBAR_EL2`/`ELR_EL2`/`SPSR_EL2`, translation walks `TTBR0_EL2`/`TCR_EL2`, and +`HCR_EL2.E2H` (VHE) redirects EL1 register names accessed from EL2 to their EL2 banks. +`system::monitor` loads Apple's SPTM monitor (`sptm.t8103.release`) the way the boot +firmware does: the image placed in DRAM at its physical address, the MMU off, EL2, and +`x0` a boot structure (DRAM's virtual and physical base and size, the device tree, a +scratch buffer). The monitor builds its own translation tables, turns the MMU on, and +checks what iBoot would have recorded in `chosen`: the `chosen/memory-map` regions, and +`dram-base` and `dram-size`. Its strings suggest it applies the kernelcache's chained +fixups itself (`sptm_fixup`); that is inferred, not confirmed. + +The region table is walked in a fixed order, and each present region must start where the +previous present one ended. In order: `TXM-ro`, `TXM-rx`, `TXM-bx`, `TrustCache`, +`AuxKC-ro/rx` (optional), `BootKC-rx`, `BootKC-bx`, `BootKC-ro`, `BootKC-rs`, +`CL4-rx/ro` (optional), `DeviceTree`, `SPTM-ro`, `SPTM-rx`, `SPTM-rw`. Each part is a +contiguous link span (`monitor::image_parts`), copied whole (`monitor::place_parts`). The +monitor's own image stays linear, because it runs position-independently before its MMU +is on. With that packing, validation passes and the monitor reaches `/chosen/dram-base`. + +It then stops in a data abort. The monitor maps physical ranges through a table of +`{VA base, PA base, pages}` entries (the PAPT table, at `0xfffffff027105000 + 0x9d0`) that its registrar +(`0xfffffff0270bd224`) fills. That function is `SPTM_FUNCTIONID_SLIDE_REGION` +(found in a dispatch table at `0xfffffff027021dd0`): it registers one physical range into +the PAPT table for callers that send the command; it checks each physical base against +`sptm_first_dram`/`sptm_last_dram`. The bootstrap does not call it. The bootstrap fills +the table in its own DRAM walk (`0xfffffff0270d05f4`, the entry writes at `0xfffffff0270d0804`), +then zeroes authored pages at `0xfffffff0270d0b0c` through the same table. The abort is a +lookup miss in that walk (`0x3fe820000000`), after which earlier notes about a +`VA & 0x0000_ffff_ffff_ffff` physical-address convention are artifacts of the miss, not a +register convention. GXF (`genter`, `gexit`, `GXF_*`, `VBAR_GL1`) and SPRR are not +modelled; `genter` raises an undefined instruction. + +Two debugging hooks make this tractable. `Machine::set_step_trace` runs one instruction +per block and reports each step with the system registers it changed, exception entries +included. `Machine::set_memory_trace` reports every translated load and store (atomics +and SIMD structure accesses excepted). Installing either turns the inline data TLB off, +so a traced run is much slower than an untraced one. + +Not modelled: any arm-io device other than the AIC, the UART and the power-state +blocks. Those addresses read as zero and writes are dropped, and the boot report counts +them as `unmodeled device accesses`, so driver probes of missing hardware (DART, GPU, +SMC, PCIe) fail rather than fault. Secondary cores stay powered off (the device tree +gives them no release address), the physical timer never fires, and fast IPIs are +storage only. + ## Performance Translating a block costs much more than running it once, and a kernel boot runs most diff --git a/src/disasm/Cargo.toml b/src/disasm/Cargo.toml new file mode 100644 index 0000000..b304d0e --- /dev/null +++ b/src/disasm/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "volt-disasm" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +path = "src/lib.rs" + +[dependencies] +object = { workspace = true, features = ["read_core", "elf", "macho", "std"] } +volt-isa-aarch64.workspace = true + +[lints.clippy] +too_many_arguments = "allow" diff --git a/src/disasm/examples/coverage.rs b/src/disasm/examples/coverage.rs new file mode 100644 index 0000000..f72f27c --- /dev/null +++ b/src/disasm/examples/coverage.rs @@ -0,0 +1,29 @@ +//! How much of an AArch64 image's executable sections the decoder recognises, and +//! which encodings it does not (the most frequent first, bucketed by opcode bits). +//! Formatting is checked separately by the `fmtdiff` example. +use std::collections::HashMap; +use volt_disasm::Image; +use volt_isa_aarch64::decode::decode; + +fn main() { + let path = std::env::args().nth(1).expect("usage: coverage "); + let img = Image::parse(&std::fs::read(path).unwrap()).unwrap(); + let (mut total, mut undecoded) = (0u64, 0u64); + let mut buckets: HashMap = HashMap::new(); + for s in img.segments.iter().filter(|s| s.exec) { + for c in s.data.chunks_exact(4) { + let w = u32::from_le_bytes(c.try_into().unwrap()); + total += 1; + if decode(w).is_err() { + undecoded += 1; + buckets.entry(w & 0xffe0_fc00).or_insert((0, w)).0 += 1; + } + } + } + println!("total {total} undecoded {undecoded}"); + let mut v: Vec<_> = buckets.into_iter().collect(); + v.sort_by_key(|x| std::cmp::Reverse(x.1.0)); + for (_, (n, w)) in v.iter().take(40) { + println!("{n:6} {w:08x}"); + } +} diff --git a/src/disasm/examples/dump.rs b/src/disasm/examples/dump.rs new file mode 100644 index 0000000..a8d740b --- /dev/null +++ b/src/disasm/examples/dump.rs @@ -0,0 +1,29 @@ +use volt_disasm::{Analysis, Image}; +fn main() { + let path = std::env::args() + .nth(1) + .expect("usage: dump [function]"); + let want = std::env::args().nth(2); + let img = Image::parse(&std::fs::read(path).unwrap()).unwrap(); + let a = Analysis::run(&img, &[]); + let blocks: usize = a.functions.values().map(|f| f.blocks.len()).sum(); + eprintln!( + "{} symbols, {} functions, {} blocks, {} xref targets", + img.symbols.len(), + a.functions.len(), + blocks, + a.xrefs.len() + ); + let Some(want) = want else { return }; + let f = a + .functions + .values() + .find(|f| f.name.as_deref() == Some(&want)) + .expect("no such function"); + for b in f.blocks.values() { + println!("block {:#x} -> {:x?} [{:?}]", b.start, b.succs, b.exit); + for i in img.disassemble(b.start, ((b.end - b.start) / 4) as usize) { + println!(" {:#x}: {:08x} {}", i.addr, i.word, i.text); + } + } +} diff --git a/src/disasm/examples/fmtdiff.rs b/src/disasm/examples/fmtdiff.rs new file mode 100644 index 0000000..651ffeb --- /dev/null +++ b/src/disasm/examples/fmtdiff.rs @@ -0,0 +1,158 @@ +//! Compare `volt_isa_aarch64::format` against GNU `objdump` over an AArch64 ELF. +//! +//! ```text +//! cargo run --release -p volt-disasm --example fmtdiff -- [mnemonic] [limit] +//! ``` +//! +//! For every instruction `objdump -d` lists it decodes the word with +//! `volt_isa_aarch64::decode`, renders it at its address, and compares the text +//! after normalising what the two spell differently on purpose (trailing `// ...` +//! comments, `` annotations, and the `0x` on branch targets). It +//! reports totals, then mismatches and undecodable words grouped by `objdump`'s +//! mnemonic, most frequent first, with one example of each. A mnemonic argument +//! restricts the report to that mnemonic and prints up to `limit` examples. + +use std::collections::HashMap; +use std::process::Command; +use volt_isa_aarch64::decode::decode; + +/// `objdump`'s own spelling, reduced to what must agree. +fn normalize(text: &str) -> String { + let text = text.split("//").next().unwrap_or(""); + let text = match text.find('<') { + Some(at) => &text[..at], + None => text, + }; + let text = text.split_whitespace().collect::>().join(" "); + let text = text.replace(", ", ","); + // A branch target is the last operand, written bare by objdump and `0x...` here. + match text.rsplit_once(',').or_else(|| text.rsplit_once(' ')) { + Some((head, last)) + if last.len() >= 4 + && !last.starts_with('#') + && last + .trim_start_matches("0x") + .chars() + .all(|c| c.is_ascii_hexdigit()) + && (last.starts_with("0x") || head.contains(' ') || !head.contains(',')) => + { + let sep = if text.contains(',') { "," } else { " " }; + format!("{head}{sep}{}", last.trim_start_matches("0x")) + } + _ => text, + } +} + +fn main() { + let mut args = std::env::args().skip(1); + let path = args + .next() + .expect("usage: fmtdiff [mnemonic] [limit]"); + let only = args.next(); + let limit: usize = args.next().map_or(3, |n| n.parse().unwrap()); + let listing = Command::new("objdump") + .args(["-d", "--no-show-raw-insn", "-w", &path]) + .output() + .expect("objdump"); + let raw = Command::new("objdump") + .args(["-d", "-w", &path]) + .output() + .expect("objdump"); + let listing = String::from_utf8_lossy(&listing.stdout).into_owned(); + let raw = String::from_utf8_lossy(&raw.stdout).into_owned(); + + // address -> objdump text, and address -> word. + let mut text: Vec<(u64, String)> = Vec::new(); + for line in listing.lines() { + let Some((addr, rest)) = line.trim().split_once(":\t") else { + continue; + }; + if let Ok(addr) = u64::from_str_radix(addr, 16) { + text.push((addr, rest.trim().to_string())); + } + } + let mut words: HashMap = HashMap::new(); + for line in raw.lines() { + let mut parts = line.trim().splitn(3, '\t'); + let (Some(addr), Some(word)) = (parts.next(), parts.next()) else { + continue; + }; + if let (Ok(addr), Ok(word)) = ( + u64::from_str_radix(addr.trim_end_matches(':'), 16), + u32::from_str_radix(word.trim(), 16), + ) { + words.insert(addr, word); + } + } + + struct Group { + count: usize, + examples: Vec<(u32, String, String)>, + } + let mut mismatch: HashMap = HashMap::new(); + let mut undecoded: HashMap = HashMap::new(); + let (mut total, mut matched) = (0usize, 0usize); + for (addr, expected) in &text { + let Some(&word) = words.get(addr) else { + continue; + }; + // Data in the text section: `.word`, `.inst`, `udf`. + let mnemonic = expected.split_whitespace().next().unwrap_or("").to_string(); + if mnemonic.starts_with('.') { + continue; + } + total += 1; + let (group, ours) = match decode(word) { + Ok(instruction) => { + let ours = instruction.display_at(*addr).to_string(); + if normalize(&ours) == normalize(expected) { + matched += 1; + continue; + } + (&mut mismatch, ours) + } + Err(_) => (&mut undecoded, String::new()), + }; + if only.as_ref().is_some_and(|m| *m != mnemonic) { + let _ = ours; + group + .entry(mnemonic) + .or_insert(Group { + count: 0, + examples: Vec::new(), + }) + .count += 1; + continue; + } + let entry = group.entry(mnemonic).or_insert(Group { + count: 0, + examples: Vec::new(), + }); + entry.count += 1; + if entry.examples.len() < limit { + entry.examples.push((word, expected.clone(), ours)); + } + } + println!( + "total {total} matched {matched} mismatched {} undecoded {}", + mismatch.values().map(|g| g.count).sum::(), + undecoded.values().map(|g| g.count).sum::() + ); + for (title, groups) in [("MISMATCH", &mismatch), ("UNDECODED", &undecoded)] { + let mut groups: Vec<_> = groups.iter().collect(); + groups.sort_by_key(|(name, g)| (std::cmp::Reverse(g.count), (*name).clone())); + for (name, group) in groups + .into_iter() + .filter(|(name, _)| only.as_ref().is_none_or(|m| m == *name)) + .take(if only.is_some() { usize::MAX } else { 40 }) + { + println!("{title} {:>6} {name}", group.count); + for (word, expected, ours) in &group.examples { + println!(" {word:08x} objdump: {expected}"); + if title == "MISMATCH" { + println!(" ours: {ours}"); + } + } + } + } +} diff --git a/src/disasm/src/analysis.rs b/src/disasm/src/analysis.rs new file mode 100644 index 0000000..1b00130 --- /dev/null +++ b/src/disasm/src/analysis.rs @@ -0,0 +1,178 @@ +//! Recursive-descent function discovery, basic blocks and cross-references. +use crate::Image; +use std::collections::{BTreeMap, BTreeSet}; +use volt_isa_aarch64::flow::{Flow, flow}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Block { + pub start: u64, + /// Address one past the last instruction. + pub end: u64, + /// Successor block starts within the same function. + pub succs: Vec, + /// How the block ends. + pub exit: Flow, +} + +#[derive(Clone, Debug)] +pub struct Function { + pub entry: u64, + pub name: Option, + pub blocks: BTreeMap, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum XrefKind { + Call, + Jump, + CondJump, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Xref { + pub from: u64, + pub kind: XrefKind, +} + +#[derive(Debug, Default)] +pub struct Analysis { + pub functions: BTreeMap, + /// Target address -> references to it, sorted. + pub xrefs: BTreeMap>, +} + +impl Analysis { + /// Discover functions from function symbols, the entry point and `extra` + /// roots, following direct calls. + pub fn run(image: &Image, extra: &[u64]) -> Self { + let mut roots: BTreeSet = image + .symbols + .iter() + .filter(|s| s.func && image.is_exec(s.addr)) + .map(|s| s.addr) + .collect(); + roots.extend(image.entry.filter(|&e| image.is_exec(e))); + roots.extend(extra.iter().copied().filter(|&a| image.is_exec(a))); + let known = roots.clone(); + + let mut out = Self::default(); + let mut queue: Vec = roots.into_iter().collect(); + while let Some(entry) = queue.pop() { + if out.functions.contains_key(&entry) { + continue; + } + let f = out.build(image, entry, &known); + for b in f.blocks.values() { + if let Flow::Call(t) = b.exit + && image.is_exec(t) + { + queue.push(t); + } + } + out.functions.insert(entry, f); + } + for v in out.xrefs.values_mut() { + v.sort(); + v.dedup(); + } + out + } + + pub fn function_containing(&self, addr: u64) -> Option<&Function> { + self.functions.values().find(|f| { + f.blocks + .range(..=addr) + .next_back() + .is_some_and(|(_, b)| addr < b.end) + }) + } + + fn build(&mut self, image: &Image, entry: u64, known: &BTreeSet) -> Function { + // Pass 1: reachable instructions and leaders. + let mut insns: BTreeMap = BTreeMap::new(); + let mut leaders = BTreeSet::from([entry]); + let mut work = vec![entry]; + while let Some(mut pc) = work.pop() { + loop { + if insns.contains_key(&pc) || !image.is_exec(pc) { + break; + } + let Some(word) = image.read_u32(pc) else { + break; + }; + let fl = flow(word, pc); + insns.insert(pc, fl); + let kind = match fl { + Flow::Call(_) => Some(XrefKind::Call), + Flow::Jump(_) => Some(XrefKind::Jump), + Flow::CondJump(_) => Some(XrefKind::CondJump), + _ => None, + }; + if let (Some(kind), Some(t)) = (kind, fl.target()) { + self.xrefs + .entry(t) + .or_default() + .push(Xref { from: pc, kind }); + } + match fl { + // A jump to another function's entry is a tail call. + Flow::Jump(t) | Flow::CondJump(t) if !(known.contains(&t) && t != entry) => { + leaders.insert(t); + work.push(t); + } + _ => {} + } + if fl.ends_block() { + if fl.falls_through() { + leaders.insert(pc + 4); + work.push(pc + 4); + } + break; + } + pc += 4; + } + } + // Pass 2: cut blocks at leaders. + let mut blocks = BTreeMap::new(); + for &start in leaders.iter().filter(|l| insns.contains_key(l)) { + let mut pc = start; + let (end, exit) = loop { + let fl = insns[&pc]; + let next = pc + 4; + if fl.ends_block() || leaders.contains(&next) || !insns.contains_key(&next) { + break (next, fl); + } + pc = next; + }; + let mut succs = Vec::new(); + if exit.falls_through() && insns.contains_key(&end) { + succs.push(end); + } + if let Flow::Jump(t) | Flow::CondJump(t) = exit + && insns.contains_key(&t) + && !(known.contains(&t) && t != entry) + { + succs.push(t); + } + blocks.insert( + start, + Block { + start, + end, + succs, + exit, + }, + ); + } + let name = image + .symbols + .iter() + .find(|s| s.addr == entry && s.func) + .map(|s| s.name.clone()); + Function { + entry, + name, + blocks, + } + } +} diff --git a/src/disasm/src/image.rs b/src/disasm/src/image.rs new file mode 100644 index 0000000..e55ea2e --- /dev/null +++ b/src/disasm/src/image.rs @@ -0,0 +1,134 @@ +//! A loaded binary: mapped segments and symbols. +use object::{Object, ObjectSection, ObjectSymbol, SectionKind, SymbolKind}; + +#[derive(Clone, Debug)] +pub struct Segment { + pub name: String, + pub addr: u64, + pub data: Vec, + pub exec: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Symbol { + pub name: String, + pub addr: u64, + pub size: u64, + pub func: bool, +} + +#[derive(Clone, Debug, Default)] +pub struct Image { + /// Sorted by address. + pub segments: Vec, + /// Sorted by address. + pub symbols: Vec, + pub entry: Option, +} + +impl Image { + /// Parse an ELF or Mach-O image. Raw files are not detected; use + /// [`Image::raw`]. + pub fn parse(bytes: &[u8]) -> Result { + let file = object::File::parse(bytes).map_err(|e| e.to_string())?; + if file.architecture() != object::Architecture::Aarch64 { + return Err(format!( + "unsupported architecture {:?}", + file.architecture() + )); + } + let mut segments = Vec::new(); + for s in file.sections() { + let exec = s.kind() == SectionKind::Text; + if !(exec + || matches!( + s.kind(), + SectionKind::Data | SectionKind::ReadOnlyData | SectionKind::ReadOnlyString + )) + { + continue; + } + let Ok(data) = s.data() else { continue }; + if data.is_empty() { + continue; + } + segments.push(Segment { + name: s.name().unwrap_or("").to_string(), + addr: s.address(), + data: data.to_vec(), + exec, + }); + } + segments.sort_by_key(|s| s.addr); + let mut symbols: Vec = file + .symbols() + .chain(file.dynamic_symbols()) + .filter(|s| s.address() != 0 && !s.name().unwrap_or("").is_empty()) + .filter(|s| { + matches!( + s.kind(), + SymbolKind::Text | SymbolKind::Data | SymbolKind::Unknown + ) + }) + .map(|s| Symbol { + name: s.name().unwrap_or("").to_string(), + addr: s.address(), + size: s.size(), + func: s.kind() == SymbolKind::Text, + }) + .collect(); + symbols.sort_by(|a, b| a.addr.cmp(&b.addr).then_with(|| a.name.cmp(&b.name))); + symbols.dedup(); + let entry = Some(file.entry()).filter(|&e| e != 0); + Ok(Self { + segments, + symbols, + entry, + }) + } + + /// An image of one executable blob at `addr`. + pub fn raw(addr: u64, code: Vec) -> Self { + Self { + segments: vec![Segment { + name: "raw".into(), + addr, + data: code, + exec: true, + }], + symbols: Vec::new(), + entry: Some(addr), + } + } + + fn segment_at(&self, addr: u64) -> Option<&Segment> { + let i = self + .segments + .partition_point(|s| s.addr <= addr) + .checked_sub(1)?; + let s = &self.segments[i]; + (addr - s.addr < s.data.len() as u64).then_some(s) + } + + pub fn is_exec(&self, addr: u64) -> bool { + self.segment_at(addr).is_some_and(|s| s.exec) + } + + pub fn read_u32(&self, addr: u64) -> Option { + let s = self.segment_at(addr)?; + let o = (addr - s.addr) as usize; + Some(u32::from_le_bytes(s.data.get(o..o + 4)?.try_into().ok()?)) + } + + /// The symbol containing `addr` (or the nearest preceding one when sizes + /// are unknown), with the offset into it. + pub fn symbolize(&self, addr: u64) -> Option<(&Symbol, u64)> { + let i = self + .symbols + .partition_point(|s| s.addr <= addr) + .checked_sub(1)?; + let s = &self.symbols[i]; + let off = addr - s.addr; + (s.size == 0 || off < s.size).then_some((s, off)) + } +} diff --git a/src/disasm/src/lib.rs b/src/disasm/src/lib.rs new file mode 100644 index 0000000..d1d31d7 --- /dev/null +++ b/src/disasm/src/lib.rs @@ -0,0 +1,62 @@ +//! Disassembly framework: binary images, linear listing, and control-flow +//! recovery over `volt-isa-aarch64`'s decoder and formatter. +pub mod analysis; +pub mod image; + +pub use analysis::{Analysis, Block, Function, Xref, XrefKind}; +pub use image::{Image, Segment, Symbol}; + +use volt_isa_aarch64::decode::decode; +use volt_isa_aarch64::flow::{Flow, flow}; +use volt_isa_aarch64::format::{self, Symbols}; + +/// One decoded instruction. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Insn { + pub addr: u64, + pub word: u32, + pub text: String, + pub flow: Flow, +} + +impl Insn { + /// Decode `word` at `addr`; targets print absolute, annotated by `symbols`. + pub fn decode(addr: u64, word: u32, symbols: &dyn Symbols) -> Self { + // One decode serves both the text and the flow; a word the decoder does not + // recognise prints raw and is classified by `flow`. + match decode(word) { + Ok(instruction) => Self { + addr, + word, + text: instruction.display_with(addr, symbols).to_string(), + flow: instruction.flow(addr), + }, + Err(_) => Self { + addr, + word, + text: format::word_with(word, addr, symbols).to_string(), + flow: flow(word, addr), + }, + } + } +} + +impl Symbols for Image { + fn lookup(&self, addr: u64) -> Option<(&str, u64)> { + self.symbolize(addr) + .map(|(s, offset)| (s.name.as_str(), offset)) + } +} + +impl Image { + /// Decode `count` instructions starting at `addr`, stopping at the end of + /// the mapped segment. + pub fn disassemble(&self, addr: u64, count: usize) -> Vec { + (0..count as u64) + .map_while(|i| { + let a = addr + i * 4; + self.read_u32(a).map(|w| Insn::decode(a, w, self)) + }) + .collect() + } +} diff --git a/src/emulator/Cargo.toml b/src/emulator/Cargo.toml index 1e4501d..cc8e9c3 100644 --- a/src/emulator/Cargo.toml +++ b/src/emulator/Cargo.toml @@ -6,7 +6,9 @@ license.workspace = true [dependencies] volt-ir.workspace = true +volt-isa-aarch64.workspace = true volt-target.workspace = true +volt-lift-aarch64.workspace = true libc = { workspace = true, optional = true } parking_lot = { workspace = true, optional = true } diff --git a/src/emulator/cli/src/boot.rs b/src/emulator/cli/src/boot.rs index 8832cc9..683fcbc 100644 --- a/src/emulator/cli/src/boot.rs +++ b/src/emulator/cli/src/boot.rs @@ -1,4 +1,5 @@ -//! Raw Linux arm64 Image, or XNU Mach-O kernel, boot through Volt's full-system JIT. +//! Raw Linux arm64 Image, XNU Mach-O kernel, or `-bios` firmware image, booted through Volt's +//! full-system JIT. use std::{ io::Write, @@ -8,8 +9,8 @@ use std::{ }; use volt_emulator::devices::BlockBackend; use volt_emulator::system::{ - DEFAULT_CMDLINE, DEFAULT_XNU_CMDLINE, IdleMode, RAM_SIZE, SystemConfig, XNU_RAM_SIZE, - smp::SmpConfig, + Board, DEFAULT_CMDLINE, DEFAULT_XNU_CMDLINE, FIRMWARE_RAM_SIZE, IdleMode, RAM_SIZE, + SystemConfig, XNU_RAM_SIZE, m1, smp::SmpConfig, }; use winnow_args::{Args, ValueEnum}; @@ -36,12 +37,12 @@ impl From for IdleMode { #[derive(Args, Debug)] #[arg( name = "volt-boot", - about = "AArch64 Linux or XNU boot with PL011, GICv2/GICv3, and VirtIO-Blk (Linux)" + about = "AArch64 Linux, XNU or firmware boot with PL011, GICv2/GICv3, VirtIO-Blk, or the Apple M1 SoC" )] struct Cli { - /// Path to a raw Linux AArch64 Image, or an XNU kernel (Mach-O). + /// Path to a raw Linux AArch64 Image, or an XNU kernel (Mach-O). Omit with --bios. #[arg(positional)] - image: PathBuf, + image: Option, /// Timeout in seconds (default: 20). #[arg(positional)] @@ -51,7 +52,8 @@ struct Cli { #[arg(positional)] cmdline: Option, - /// Disk image for virtio-blk rootfs (e.g. rootfs.ext4, disk.img). + /// Disk image (e.g. rootfs.ext4, disk.img). A kernel boot gets it as virtio-blk on MMIO; a + /// `--bios` boot as virtio-blk on PCI, at 00:01.0. #[arg(short, long, alias = "drive", alias = "rootfs")] disk: Option, @@ -59,6 +61,20 @@ struct Cli { #[arg(short, long, alias = "initramfs")] initrd: Option, + /// Device tree for the Apple M1 (t8103) SoC, such as the Asahi kernel's + /// `t8103-j274.dtb`. Selects that machine in place of the QEMU `virt` board. + #[arg(long)] + dtb: Option, + + /// Firmware image to run from flash at address 0, as QEMU's `-bios` does (e.g. an EDK2 + /// `QEMU_EFI.fd` or tinted-boot's `tinted-boot-aarch64.bin`). Boots no kernel. + #[arg(long)] + bios: Option, + + /// Timeout in seconds, for any boot (default: 20). Alternative to the seconds positional. + #[arg(long)] + timeout: Option, + /// Guest RAM size (e.g. 128M, 256M, 512M, 1G). #[arg(short, long, alias = "ram")] memory: Option, @@ -156,6 +172,12 @@ fn main() -> ExitCode { } else if s == "-smp" { args.push("--smp".into()); continue; + } else if s == "-bios" { + args.push("--bios".into()); + continue; + } else if let Some(rest) = s.strip_prefix("-bios=") { + args.push(format!("--bios={rest}").into()); + continue; } } args.push(arg); @@ -191,12 +213,27 @@ fn run(cli: Cli) -> Result<(), String> { IdleMode::Paced }; - let seconds = cli.seconds.unwrap_or(20); - let image = std::fs::read(&cli.image).map_err(|e| format!("{}: {e}", cli.image.display()))?; + let seconds = cli.timeout.or(cli.seconds).unwrap_or(20); + if let Some(bios) = &cli.bios { + return run_firmware(&cli, bios, idle_mode, seconds); + } + let image_path = cli + .image + .as_ref() + .ok_or("no kernel image given: pass a Linux Image or XNU kernel, or --bios FIRMWARE")?; + let image = std::fs::read(image_path).map_err(|e| format!("{}: {e}", image_path.display()))?; + let board = match &cli.dtb { + Some(path) => Board::AppleM1 { + device_tree: std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?, + }, + None => Board::Virt, + }; // A Mach-O image is an XNU kernel; anything else is a raw Linux Image. let xnu = volt_emulator::system::xnu::is_macho(&image); let default_cmdline = if xnu { DEFAULT_XNU_CMDLINE + } else if matches!(board, Board::AppleM1 { .. }) { + m1::DEFAULT_CMDLINE } else if cli.disk.is_some() { "root=/dev/vda rw console=ttyAMA0 earlycon=pl011,0x9000000 nokaslr loglevel=8" } else { @@ -213,6 +250,8 @@ fn run(cli: Cli) -> Result<(), String> { RAM_SIZE.max((initrd.len() * 3 + (128 << 20)).next_power_of_two()) } else if xnu { XNU_RAM_SIZE + } else if matches!(board, Board::AppleM1 { .. }) { + m1::DEFAULT_RAM_SIZE } else { RAM_SIZE }; @@ -227,6 +266,9 @@ fn run(cli: Cli) -> Result<(), String> { None => None, }; + if cli.cpus > 1 && !matches!(board, Board::Virt) { + return Err("the Apple M1 SoC boots one vCPU; pass --smp 1".into()); + } if cli.cpus > 1 { let config = SmpConfig { cpus: cli.cpus, @@ -256,6 +298,7 @@ fn run(cli: Cli) -> Result<(), String> { ram_size, initrd: initrd_data, disk: disk_backend, + board, }; let mut output = std::io::BufWriter::new(std::io::stdout()); let report = volt_emulator::system::boot_system(&image, config, move |byte| { @@ -269,6 +312,54 @@ fn run(cli: Cli) -> Result<(), String> { Ok(()) } +/// Boots a firmware image as QEMU's `-bios` does: no kernel, the image run from flash at 0. +fn run_firmware(cli: &Cli, bios: &Path, idle_mode: IdleMode, seconds: u64) -> Result<(), String> { + if cli.image.is_some() { + return Err( + "--bios boots firmware without a kernel; drop the positional arguments and use --timeout" + .into(), + ); + } + if cli.dtb.is_some() { + return Err("--bios and --dtb select different machines".into()); + } + if cli.cpus > 1 { + return Err("firmware boot has one vCPU for now; pass --smp 1".into()); + } + let firmware = std::fs::read(bios).map_err(|e| format!("{}: {e}", bios.display()))?; + let ram_size = match cli.memory.as_deref() { + Some(s) => parse_memory_size(s)?, + None => FIRMWARE_RAM_SIZE, + }; + let initrd = match &cli.initrd { + Some(path) => Some(load_initrd(path)?), + None => None, + }; + let disk = match &cli.disk { + Some(path) => Some(FileBackend::open(path)?), + None => None, + }; + let config = SystemConfig { + timeout: Duration::from_secs(seconds), + cmdline: String::new(), + idle_mode, + ram_size, + initrd, + disk, + board: Board::Virt, + }; + let mut output = std::io::BufWriter::new(std::io::stdout()); + let report = volt_emulator::system::boot_firmware(&firmware, config, move |byte| { + let _ = output.write_all(&[byte]); + if byte == b'\n' { + let _ = output.flush(); + } + }) + .map_err(|e| format!("{e}"))?; + println!("\n{report}"); + Ok(()) +} + fn load_initrd(path: &Path) -> Result, String> { let bytes = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?; if bytes.len() >= 4 && bytes[..4] == [0x28, 0xb5, 0x2f, 0xfd] { @@ -349,6 +440,18 @@ mod tests { assert_eq!(cli.initrd, Some(PathBuf::from("initramfs.cpio"))); } + #[test] + fn test_winnow_args_bios_needs_no_kernel() { + let cli = Cli::parse_from(["--bios", "tinted-boot-aarch64.bin", "--timeout", "7"]).unwrap(); + assert_eq!(cli.bios, Some(PathBuf::from("tinted-boot-aarch64.bin"))); + assert!(cli.image.is_none()); + assert_eq!(cli.timeout, Some(7)); + + let cli = Cli::parse_from(["Image"]).unwrap(); + assert!(cli.bios.is_none()); + assert_eq!(cli.image, Some(PathBuf::from("Image"))); + } + #[test] fn test_zstd_decompression() { let data = b"sample cpio content for zstd test"; diff --git a/src/emulator/examples/jit_bench.rs b/src/emulator/examples/jit_bench.rs index 9c27285..413151b 100644 --- a/src/emulator/examples/jit_bench.rs +++ b/src/emulator/examples/jit_bench.rs @@ -20,14 +20,14 @@ fn block(n: usize, seed: u32) -> Vec { fn main() { dispatch(); { - use volt_emulator::aarch64::decode::decode; + use volt_emulator::aarch64::host::ends_block; + use volt_isa_aarch64::decode::decode; let b = block(63, 1); let n = b .chunks_exact(4) .take_while(|c| { - !decode(u32::from_le_bytes((*c).try_into().unwrap())) - .unwrap() - .terminates() + let word = u32::from_le_bytes((*c).try_into().unwrap()); + !ends_block(&decode(word).unwrap(), false) }) .count(); println!("non-terminating prefix of 63-block: {n}"); diff --git a/src/emulator/src/aarch64/ahead.rs b/src/emulator/src/aarch64/ahead.rs index 6b34624..f4c97d0 100644 --- a/src/emulator/src/aarch64/ahead.rs +++ b/src/emulator/src/aarch64/ahead.rs @@ -19,12 +19,16 @@ //! on a worker thread. use super::cache::{MAX_INSNS, SharedBlocks}; -use super::decode::{Instruction, decode}; +use super::host::ends_block; use alloc::collections::VecDeque; use alloc::sync::{Arc, Weak}; use std::collections::HashSet; use std::sync::{Condvar, Mutex, MutexGuard, PoisonError}; use std::thread; +use volt_isa_aarch64::{ + decode::{Instruction, decode}, + flow::Flow, +}; /// Reads guest-physical bytes without any vCPU state, for worker threads. pub trait CodeReader: Send + Sync { @@ -66,20 +70,15 @@ fn lock(state: &Mutex) -> MutexGuard<'_, State> { /// Where control can go after `last`, a block's final instruction at `last_pc`. /// Fixed targets only: an indirect branch or return has none to offer. -fn successors(last: Instruction, last_pc: u64) -> ([u64; 2], usize) { +fn successors(instruction: Instruction, last_pc: u64) -> ([u64; 2], usize) { let next = last_pc.wrapping_add(4); - let at = |offset: i64| last_pc.wrapping_add(offset as u64); - match last { - Instruction::B(offset) => ([at(offset), 0], 1), - Instruction::Call(call) if call.link => ([at(call.target), next], 2), - Instruction::Call(call) => ([at(call.target), 0], 1), - Instruction::BCond(branch) => ([at(branch.offset), next], 2), - Instruction::TestBranch(test) => ([at(test.offset), next], 2), - Instruction::Indirect(branch) if branch.link => ([next, 0], 1), - Instruction::Indirect(_) | Instruction::Eret => ([0, 0], 0), - // Every other block end, a trap or a cut at the instruction limit, resumes - // at the next instruction. - _ => ([next, 0], 1), + match instruction.flow(last_pc) { + Flow::Jump(target) => ([target, 0], 1), + Flow::CondJump(target) | Flow::Call(target) => ([target, next], 2), + Flow::JumpIndirect | Flow::Return => ([0, 0], 0), + // Every other block end, a call returning, a trap or a cut at the + // instruction limit, resumes at the next instruction. + Flow::Next | Flow::CallIndirect | Flow::Stop | Flow::Unrecognized => ([next, 0], 1), } } @@ -187,7 +186,7 @@ impl Ahead { }; last = Some((instruction, job.pc + count as u64)); count += 4; - if instruction.terminates_with(shared.inline_memory()) + if ends_block(&instruction, shared.inline_memory()) || n + 1 == MAX_INSNS || (job.pc + count as u64) & 0xfff == 0 { @@ -259,7 +258,7 @@ mod tests { #[test] fn fixed_targets_follow_each_kind_of_block_end() { - use crate::aarch64::decode::{Call, Test, Width}; + use volt_isa_aarch64::decode::{Call, Indirect, IndirectKind, Test, Width}; let at = 0x1000; let test = Instruction::TestBranch(Test { rt: 0, @@ -275,15 +274,21 @@ mod tests { (call(false), vec![0x1040]), (test, vec![0x0ff8, 0x1004]), ( - Instruction::Indirect(crate::aarch64::decode::Indirect { rn: 1, link: false }), + Instruction::Indirect(Indirect { + rn: 1, + kind: IndirectKind::Jump, + }), vec![], ), ( - Instruction::Indirect(crate::aarch64::decode::Indirect { rn: 1, link: true }), + Instruction::Indirect(Indirect { + rn: 1, + kind: IndirectKind::Call, + }), vec![0x1004], ), (Instruction::Eret, vec![]), - (Instruction::Svc, vec![0x1004]), + (Instruction::Svc(0), vec![0x1004]), (Instruction::Nop, vec![0x1004]), ]; for (last, expected) in cases { diff --git a/src/emulator/src/aarch64/cache.rs b/src/emulator/src/aarch64/cache.rs index 350fbc7..a93e0f9 100644 --- a/src/emulator/src/aarch64/cache.rs +++ b/src/emulator/src/aarch64/cache.rs @@ -2,7 +2,7 @@ use super::{ ahead::{Ahead, CodeReader}, compile::{Block, Error, compile_with}, cpu::Cpu, - decode::decode, + host::ends_block, }; use core::hash::{Hash, Hasher}; use core::sync::atomic::{AtomicUsize, Ordering}; @@ -26,6 +26,8 @@ pub struct Cache { /// Where blocks come from on a local miss. Private to this cache unless /// built with `with_shared`. shared: Arc, + /// Instructions one block scan may take: `MAX_INSNS`, or 1 when single-stepping. + max_insns: usize, } impl Default for Cache { fn default() -> Self { @@ -34,6 +36,7 @@ impl Default for Cache { index: HashMap::new(), direct: alloc_direct(), shared: Arc::new(SharedBlocks::new(false)), + max_insns: MAX_INSNS, } } } @@ -192,6 +195,17 @@ impl Cache { pub fn new() -> Self { Self::default() } + + /// A cache whose blocks are one instruction each, so a trace sees every step. + pub fn single_step(inline_memory: bool) -> Self { + let mut cache = if inline_memory { + Self::with_inline_memory() + } else { + Self::new() + }; + cache.max_insns = 1; + cache + } /// Whether blocks from this cache read `Cpu::dtlb`. pub fn inline_memory(&self) -> bool { self.shared.inline_memory() @@ -326,7 +340,7 @@ impl Cache { let mut count = 0usize; let mut complete = false; let mut last = None; - for n in 0..MAX_INSNS { + for n in 0..self.max_insns { let at = cpu .pc .checked_add(count as u64) @@ -342,7 +356,7 @@ impl Cache { } count += 4; let word = u32::from_le_bytes(room.try_into().unwrap()); - let instruction = match decode(word) { + let instruction = match volt_isa_aarch64::decode::decode(word) { Ok(insn) => insn, // Run the decoded prefix first: the undecodable word then heads the // next block, where it is reported at its own address. @@ -352,21 +366,21 @@ impl Cache { } Err(_) => { // Architecturally undefined words are the guest's to handle. - if !super::decode::is_undefined(word) { + if !volt_isa_aarch64::decode::is_undefined(word) { eprintln!("cache decode failure at pc {at:x}: word {word:08x}"); } return Err(Error::Decode { word, pc: at }); } }; last = Some((instruction, at)); - if instruction.terminates_with(self.shared.inline_memory()) { + if ends_block(&instruction, self.shared.inline_memory()) { complete = true; break; } // Never straddle a page: a block inside one page is validated by // that page's version alone, and the next page is translated and // checked when its own block starts. - complete = n + 1 == MAX_INSNS || (pc + count as u64) & 0xfff == 0; + complete = n + 1 == self.max_insns || (pc + count as u64) & 0xfff == 0; if complete { break; } diff --git a/src/emulator/src/aarch64/compile.rs b/src/emulator/src/aarch64/compile.rs index b1365ad..1e7f56e 100644 --- a/src/emulator/src/aarch64/compile.rs +++ b/src/emulator/src/aarch64/compile.rs @@ -1,17 +1,12 @@ use super::{ - cpu::{Cpu, DTLB_ENTRIES, Exclusive as CpuExclusive, System as CpuSystem, Trap}, - decode::*, + cpu::{Cpu, Trap}, + environment::CpuEnvironment, + host::{ends_block, runs_on_host}, }; use crate::memory::MemoryError; -use core::{ - cell::{Cell, RefCell}, - fmt, - mem::offset_of, -}; -use volt_ir::{ - function::{self as ir, BinOp as B, CmpOp as C, Function, Opcode, Value}, - types::{IntDesc, Type, TypeKind}, -}; +use core::fmt; +use volt_isa_aarch64::decode::decode; +use volt_lift_aarch64::{LiftError, Lifter}; use volt_target::native; #[derive(Debug)] @@ -24,6 +19,7 @@ pub enum Error { UnsupportedInstruction, AddressOverflow, InvalidInstructionLength, + Lift(LiftError), } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -38,6 +34,7 @@ impl fmt::Display for Error { Self::UnsupportedInstruction => f.write_str("unsupported instruction encoding"), Self::AddressOverflow => f.write_str("instruction address overflow"), Self::InvalidInstructionLength => f.write_str("invalid instruction fetch length"), + Self::Lift(e) => write!(f, "{e}"), } } } @@ -47,6 +44,11 @@ impl From for Error { Self::Memory(e) } } +impl From for Error { + fn from(e: LiftError) -> Self { + Self::Lift(e) + } +} impl From for Error { fn from(e: native::Error) -> Self { Self::Native(e) @@ -75,571 +77,13 @@ impl Block { next } } -struct Lower { - f: RefCell, - /// Block currently being emitted into. Memory fast paths switch it. - block: Cell, - cpu: Value, - u64: Type, - u32: Type, - u8: Type, - boolean: Type, - ptr: Type, - /// Guest instructions started but not yet added to `cntvct_el0`. The - /// counter is bumped once per exit path (`ret`) and before anything that - /// reads it, instead of with a load/add/store per instruction. - pending: Cell, -} -impl Lower { - fn new() -> Self { - let mut f = Function::new(); - // A typical block is a few blocks and about a hundred instructions. - f.reserve(8, 128, 128); - let u64 = f.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits: 64, - })); - let u32 = f.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits: 32, - })); - let u8 = f.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits: 8, - })); - let boolean = f.types.intern(TypeKind::Bool); - let ptr = f.types.ptr_global(); - let block = f.append_block(); - let cpu = f.append_block_param(block, ptr); - Self { - f: RefCell::new(f), - block: Cell::new(block), - cpu, - u64, - u32, - u8, - boolean, - ptr, - pending: Cell::new(0), - } - } - fn emit(&self, t: Type, op: Opcode) -> Value { - self.f.borrow_mut().append_inst(self.block.get(), t, op) - } - fn k(&self, t: Type, n: u64) -> Value { - self.emit(t, Opcode::Iconst(n as i64)) - } - fn bin(&self, t: Type, op: B, a: Value, b: Value) -> Value { - self.emit(t, Opcode::Arith(ir::Arith { op, lhs: a, rhs: b })) - } - fn imm(&self, t: Type, op: B, a: Value, n: u64) -> Value { - // `ArithImm` instead of `Arith` over an `Iconst`: the backend encodes - // small immediates directly and `addrfold` only folds `ArithImm` adds - // into load/store addressing. - self.emit( - t, - Opcode::ArithImm(ir::ArithImm { - op, - lhs: a, - imm: n as i64, - }), - ) - } - fn cv(&self, t: Type, v: Value) -> Value { - let same = { self.f.borrow().value_type(v) == t }; - if same { - v - } else { - self.emit(t, Opcode::Convert(ir::Convert { value: v })) - } - } - fn cmp(&self, op: C, a: Value, b: Value) -> Value { - self.emit( - self.boolean, - Opcode::Icmp(ir::Compare { op, lhs: a, rhs: b }), - ) - } - fn sel(&self, t: Type, c: Value, a: Value, b: Value) -> Value { - self.emit( - t, - Opcode::Select(ir::Select { - cond: c, - then: a, - else_: b, - }), - ) - } - fn addr(&self, at: usize) -> Value { - self.imm(self.ptr, B::Add, self.cpu, at as u64) - } - fn load_ptr(&self, t: Type, p: Value) -> Value { - self.emit( - t, - Opcode::Load(ir::Load { - ptr: p, - volatile: false, - }), - ) - } - fn load(&self, t: Type, at: usize) -> Value { - self.load_ptr(t, self.addr(at)) - } - fn store_ptr(&self, p: Value, v: Value) { - self.f.borrow_mut().append_store(self.block.get(), v, p) - } - fn store(&self, at: usize, v: Value) { - self.store_ptr(self.addr(at), v) - } - fn byte(&self, at: usize, v: u64) { - self.store(at, self.k(self.u8, v)) - } - fn reg_at(r: u8) -> usize { - if r == 31 { - offset_of!(Cpu, sp) - } else { - offset_of!(Cpu, x) + r as usize * 8 - } - } - fn reg(&self, t: Type, r: u8, zero: bool) -> Value { - if zero && r == 31 { - self.k(t, 0) - } else { - self.cv(t, self.load(self.u64, Self::reg_at(r))) - } - } - fn put(&self, r: u8, v: Value) { - if r != 31 { - self.put_sp(r, v) - } - } - fn put_sp(&self, r: u8, v: Value) { - self.store(Self::reg_at(r), self.cv(self.u64, v)) - } - fn ty(&self, w: Width) -> Type { - if w == Width::X64 { self.u64 } else { self.u32 } - } - fn signed(&self, w: Width) -> Type { - self.f.borrow_mut().types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: w as u16, - })) - } - fn add_counter(&self, n: u64) { - let at = offset_of!(Cpu, system) + offset_of!(CpuSystem, cntvct_el0); - self.store(at, self.imm(self.u64, B::Add, self.load(self.u64, at), n)); - } - /// Make `cntvct_el0` in memory current on the path being emitted. - fn flush_counter(&self) { - let n = self.pending.replace(0); - if n != 0 { - self.add_counter(n); - } - } - fn ret(&self, v: Value) { - // Every exit leaves with the counter current. `pending` is not - // cleared: sibling paths (slow blocks) share the main path's count. - let n = self.pending.get(); - if n != 0 { - self.add_counter(n); - } - self.f - .borrow_mut() - .set_terminator(self.block.get(), ir::Terminator::Ret(ir::Ret::one(v))) - } - fn not(&self, v: Value) -> Value { - self.imm(self.boolean, B::BitXor, v, 1) - } - fn flag(&self, flags: Value, bit: u32) -> Value { - self.cmp( - C::Ne, - self.imm(self.u32, B::BitAnd, flags, 1 << bit), - self.k(self.u32, 0), - ) - } - fn condition(&self, c: Condition) -> Value { - let f = self.load(self.u32, offset_of!(Cpu, flags)); - let n = self.flag(f, 31); - let z = self.flag(f, 30); - let carry = self.flag(f, 29); - let v = self.flag(f, 28); - let changed = self.bin(self.boolean, B::BitXor, n, v); - let ordered = self.not(changed); - let positive = self.not(z); - match c { - Condition::Eq => z, - Condition::Ne => positive, - Condition::Cs => carry, - Condition::Cc => self.not(carry), - Condition::Mi => n, - Condition::Pl => self.not(n), - Condition::Vs => v, - Condition::Vc => self.not(v), - Condition::Hi => self.bin(self.boolean, B::BitAnd, carry, positive), - Condition::Ls => self.not(self.bin(self.boolean, B::BitAnd, carry, positive)), - Condition::Ge => ordered, - Condition::Lt => changed, - Condition::Gt => self.bin(self.boolean, B::BitAnd, positive, ordered), - Condition::Le => self.not(self.bin(self.boolean, B::BitAnd, positive, ordered)), - Condition::Al | Condition::Nv => self.k(self.boolean, 1), - } - } - fn pack(&self, n: Value, z: Value, c: Value, v: Value) -> Value { - let mut out = self.k(self.u32, 0); - for (bit, shift) in [(n, 31), (z, 30), (c, 29), (v, 28)] { - out = self.bin( - self.u32, - B::BitOr, - out, - self.imm(self.u32, B::Shl, self.cv(self.u32, bit), shift), - ); - } - out - } - fn flags( - &self, - w: Width, - op: ArithOp, - a: Value, - b: Value, - r: Value, - carry: Option, - ) -> Value { - let t = self.ty(w); - let zero = self.k(t, 0); - let n = self.cmp( - C::Ne, - self.imm(t, B::BitAnd, r, 1u64 << ((w as u32) - 1)), - zero, - ); - let z = self.cmp(C::Eq, r, zero); - let c = carry.unwrap_or_else(|| { - if op == ArithOp::Sub { - self.cmp(C::Ge, a, b) - } else { - self.cmp(C::Lt, r, a) - } - }); - let s = self.signed(w); - let sz = self.k(s, 0); - let an = self.cmp(C::Lt, self.cv(s, a), sz); - let bn = self.cmp(C::Lt, self.cv(s, b), sz); - let rn = self.cmp(C::Lt, self.cv(s, r), sz); - let diff = self.bin(self.boolean, B::BitXor, an, bn); - let flip = self.bin(self.boolean, B::BitXor, an, rn); - let v = self.bin( - self.boolean, - B::BitAnd, - if op == ArithOp::Sub { - diff - } else { - self.not(diff) - }, - flip, - ); - self.pack(n, z, c, v) - } - fn logic_flags(&self, w: Width, r: Value) { - let t = self.ty(w); - let zero = self.k(t, 0); - let n = self.cmp( - C::Ne, - self.imm(t, B::BitAnd, r, 1u64 << ((w as u32) - 1)), - zero, - ); - let z = self.cmp(C::Eq, r, zero); - let no = self.k(self.boolean, 0); - self.store(offset_of!(Cpu, flags), self.pack(n, z, no, no)); - } - fn shift(&self, w: Width, v: Value, s: Shift, n: u64) -> Value { - if n == 0 { - return v; - } - let t = self.ty(w); - match s { - Shift::Lsl => self.imm(t, B::Shl, v, n), - Shift::Lsr => self.imm(t, B::Shr, v, n), - Shift::Asr => { - let signed = self.signed(w); - self.cv(t, self.imm(signed, B::Shr, self.cv(signed, v), n)) - } - Shift::Ror => self.bin( - t, - B::BitOr, - self.imm(t, B::Shr, v, n), - self.imm(t, B::Shl, v, w as u64 - n), - ), - } - } - fn extend(&self, v: Value, e: Extend) -> Value { - match e { - Extend::Uxtx | Extend::Sxtx => v, - Extend::Uxtb => self.imm(self.u64, B::BitAnd, v, 0xff), - Extend::Uxth => self.imm(self.u64, B::BitAnd, v, 0xffff), - Extend::Uxtw => self.imm(self.u64, B::BitAnd, v, 0xffffffff), - Extend::Sxtb | Extend::Sxth | Extend::Sxtw => { - let n = match e { - Extend::Sxtb => 56, - Extend::Sxth => 48, - _ => 32, - }; - let raised = self.imm(self.u64, B::Shl, v, n); - self.shift(Width::X64, raised, Shift::Asr, n) - } - } - } - fn address(&self, rn: u8, a: Addressing) -> Value { - let base = self.reg(self.u64, rn, false); - match a { - Addressing::Offset(n) => self.imm(self.u64, B::Add, base, n as u64), - Addressing::PreIndex(n) => { - let a = self.imm(self.u64, B::Add, base, n as u64); - self.put_sp(rn, a); - a - } - Addressing::PostIndex(n) => { - self.store( - offset_of!(Cpu, writeback_value), - self.imm(self.u64, B::Add, base, n as u64), - ); - self.byte(offset_of!(Cpu, writeback_dest), rn as u64); - self.byte(offset_of!(Cpu, writeback), 1); - base - } - Addressing::Register { rm, extend, amount } => { - let v = self.extend(self.reg(self.u64, rm, true), extend); - self.bin( - self.u64, - B::Add, - base, - self.imm(self.u64, B::Shl, v, amount as u64), - ) - } - } - } - fn branch(&self, pc: u64, offset: i64, c: Value) { - self.ret(self.sel( - self.u64, - c, - self.k(self.u64, pc.wrapping_add(offset as u64)), - self.k(self.u64, pc.wrapping_add(4)), - )) - } - fn uint(&self, bits: u16) -> Type { - self.f.borrow_mut().types.intern(TypeKind::Int(IntDesc { - signed: false, - bits, - })) - } - fn new_block(&self) -> ir::Block { - self.f.borrow_mut().append_block() - } - fn branch_if(&self, c: Value, then: ir::Block, otherwise: ir::Block) { - self.f.borrow_mut().append_if( - self.block.get(), - c, - ir::EdgeDesc::bare(then), - ir::EdgeDesc::bare(otherwise), - ) - } - /// Like `address`, but a base-register writeback (pre- or post-index) is - /// returned instead of applied or scheduled for the machine, so each path - /// can apply it its own way once the access has succeeded. An access that - /// faults must leave the base register untouched so the retry sees it. - fn address_split(&self, rn: u8, a: Addressing) -> (Value, Option<(u8, Value)>) { - match a { - Addressing::PostIndex(n) => { - let base = self.reg(self.u64, rn, false); - (base, Some((rn, self.imm(self.u64, B::Add, base, n as u64)))) - } - Addressing::PreIndex(n) => { - let base = self.reg(self.u64, rn, false); - let address = self.imm(self.u64, B::Add, base, n as u64); - (address, Some((rn, address))) - } - other => (self.address(rn, other), None), - } - } - /// Probe the inline data TLB for `address`. Returns `(hit, host_pointer)`; - /// the pointer is only meaningful when `hit`. A hit requires the page to be - /// cached for this access kind and the access to be naturally aligned (so it - /// cannot cross a page), because the tag compare covers the low `size - 1` - /// address bits too. - fn dtlb_probe(&self, address: Value, size: Size, write: bool) -> (Value, Value) { - let n = size as u64; - // Entry offset = ((address >> 12) & (ENTRIES - 1)) * 32 = (address >> 7) & mask. - let index = self.imm( - self.u64, - B::BitAnd, - self.imm(self.u64, B::Shr, address, 7), - (DTLB_ENTRIES as u64 - 1) << 5, - ); - let entry = self.bin(self.ptr, B::Add, self.cpu, index); - // Constant displacements fold into the load addressing. - let field = |at: usize| { - self.load_ptr( - self.u64, - self.imm(self.ptr, B::Add, entry, (offset_of!(Cpu, dtlb) + at) as u64), - ) - }; - let tag = field(if write { 8 } else { 0 }); - let addend = field(16); - let host = self.bin(self.ptr, B::Add, address, addend); - let want = self.imm(self.u64, B::BitAnd, address, !0xfffu64 | (n - 1)); - // Last, so the backend can fuse it into the branch that consumes it. - let hit = self.cmp(C::Eq, tag, want); - (hit, host) - } - fn fast_load(&self, host: Value, size: Size, rt: u8, signed: SignExtend) { - let bits = size as u16 * 8; - let raw = self.load_ptr(self.uint(bits), host); - if rt == 31 { - return; - } - let mut v = self.cv(self.u64, raw); - if signed != SignExtend::None && bits < 64 { - let n = 64 - bits as u64; - let extended = self.shift(Width::X64, self.imm(self.u64, B::Shl, v, n), Shift::Asr, n); - v = if signed == SignExtend::To32 { - self.imm(self.u64, B::BitAnd, extended, 0xffff_ffff) - } else { - extended - }; - } - self.put(rt, v); - } - fn fast_store(&self, host: Value, size: Size, rt: u8) { - let t = self.uint(size as u16 * 8); - self.store_ptr(host, self.cv(t, self.reg(self.u64, rt, true))); - } - /// Emit a load or store with an inline fast path. On a data-TLB hit the - /// access happens in place and emission continues in a fresh block; on a - /// miss control goes to a slow block that hands the access to the machine - /// (`slow` fills in the request) and returns. - fn memory_access( - &self, - address: Value, - size: Size, - op: MemoryOp, - rt: u8, - signed: SignExtend, - post: Option<(u8, Value)>, - slow: impl FnOnce(&Lower), - ) { - let write = op == MemoryOp::Store; - let (hit, host) = self.dtlb_probe(address, size, write); - let (fast_block, slow_block) = (self.new_block(), self.new_block()); - self.branch_if(hit, fast_block, slow_block); - self.block.set(slow_block); - slow(self); - self.block.set(fast_block); - if write { - self.fast_store(host, size, rt); - } else { - self.fast_load(host, size, rt, signed); - } - if let Some((rn, value)) = post { - self.put_sp(rn, value); - } - } - /// Inline fast path for a GPR load/store pair: one data-TLB probe covers - /// both accesses when the first is naturally aligned and the second starts - /// in the same page. Anything else takes `slow`, which hands the whole pair - /// to the machine. - fn pair_access( - &self, - address: Value, - a: Pair, - post: Option<(u8, Value)>, - slow: impl FnOnce(&Lower), - ) { - let write = a.op == MemoryOp::Store; - let n = a.size as u64; - let (hit, host) = self.dtlb_probe(address, a.size, write); - let (check_block, fast_block, slow_block) = - (self.new_block(), self.new_block(), self.new_block()); - self.branch_if(hit, check_block, slow_block); - // `address` is aligned to `n`, so the second access leaves the page - // exactly when it starts on a page boundary. - self.block.set(check_block); - let second_at = self.imm(self.u64, B::Add, address, n); - let offset = self.imm(self.u64, B::BitAnd, second_at, 0xfff); - let same_page = self.cmp(C::Ne, offset, self.k(self.u64, 0)); - self.branch_if(same_page, fast_block, slow_block); - self.block.set(slow_block); - slow(self); - self.block.set(fast_block); - let second = self.imm(self.ptr, B::Add, host, n); - if write { - self.fast_store(host, a.size, a.rt); - self.fast_store(second, a.size, a.rt2); - } else { - self.fast_load(host, a.size, a.rt, a.signed); - self.fast_load(second, a.size, a.rt2, a.signed); - } - if let Some((rn, value)) = post { - self.put_sp(rn, value); - } - } - fn trap(&self, pc: u64, t: Trap) { - self.byte(offset_of!(Cpu, trap), t as u64); - self.ret(self.k(self.u64, pc)) - } - - /// Slow path for a whole SIMD vector or vector-pair access: describe the - /// full instruction in the CPU request fields (`vector_dest`, optionally - /// `second_*`) and trap out; the machine completes every byte and resumes - /// after the instruction. - fn simd_trap_access( - &self, - address: Value, - bytes: u64, - rt: u8, - rt2: Option, - post: Option<(u8, Value)>, - pc: u64, - trap: Trap, - ) { - self.store(offset_of!(Cpu, address), address); - self.byte(offset_of!(Cpu, width), bytes); - self.byte(offset_of!(Cpu, dest), rt as u64); - self.byte(offset_of!(Cpu, vector_dest), 1); - if let Some(rt2) = rt2 { - let second = self.imm(self.u64, B::Add, address, if bytes == 16 { 16 } else { 8 }); - self.store(offset_of!(Cpu, second_address), second); - self.byte(offset_of!(Cpu, second_width), bytes); - self.byte(offset_of!(Cpu, second_dest), rt2 as u64); - self.byte(offset_of!(Cpu, second_vector), 1); - self.byte(offset_of!(Cpu, second_pending), 1); - } - if let Some((rn, value)) = post { - self.store(offset_of!(Cpu, writeback_value), value); - self.byte(offset_of!(Cpu, writeback_dest), rn as u64); - self.byte(offset_of!(Cpu, writeback), 1); - } - self.trap(pc.wrapping_add(4), trap); - } -} -fn arithmetic_op(op: ArithOp) -> B { - if op == ArithOp::Sub { B::Sub } else { B::Add } -} -fn logic_op(op: LogicOp) -> B { - match op { - LogicOp::BitAnd => B::BitAnd, - LogicOp::BitOr => B::BitOr, - LogicOp::BitXor => B::BitXor, - } -} -fn low_mask(n: u8) -> u64 { - if n >= 64 { u64::MAX } else { (1u64 << n) - 1 } -} - pub fn compile(guest_pc: u64, bytes: &[u8]) -> Result { compile_with(guest_pc, bytes, false) } /// Compile a block. With `inline_memory`, plain loads and stores get an inline /// data-TLB fast path and do not end the block; the caller's block scan must -/// use `Instruction::terminates_with(true)` to match. +/// use `host::ends_block(_, true)` to match. pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result { if bytes.is_empty() { return Err(Error::EmptyBlock); @@ -647,14 +91,22 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< if bytes.len() % 4 != 0 { return Err(Error::UnsupportedInstruction); } - let l = Lower::new(); + let mut lifter = Lifter::new(CpuEnvironment::new(inline_memory)); let mut pc = guest_pc; let mut ended = false; for chunk in bytes.chunks_exact(4) { - l.pending.set(l.pending.get() + 1); let word = u32::from_le_bytes(chunk.try_into().unwrap()); - let instruction = match decode(word) { - Ok(insn) => insn, + match decode(word) { + Ok(instruction) if runs_on_host(&instruction) => { + // The host runs this word (see `host::run`), and it ends the block. + let (builder, env) = lifter.parts(); + env.host(builder, pc, word); + ended = true; + } + Ok(instruction) => { + ended = ends_block(&instruction, inline_memory); + lifter.lift(pc, instruction)?; + } Err(_) => { eprintln!("compile decode failure at pc {pc:x}: word {word:08x}, full block:"); for (i, c) in bytes.chunks_exact(4).enumerate() { @@ -668,1496 +120,6 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } return Err(Error::Decode { word, pc }); } - }; - ended = instruction.terminates_with(inline_memory); - match instruction { - Instruction::Movz(a) | Instruction::Movn(a) => { - let t = l.ty(a.width); - let bits = (a.immediate as u64) << a.shift; - let value = if matches!(instruction, Instruction::Movn(_)) { - !bits - } else { - bits - }; - if a.rd != 31 { - l.put(a.rd, l.k(t, value)); - } - } - Instruction::Movk(a) => { - if a.rd != 31 { - let t = l.ty(a.width); - let old = l.reg(t, a.rd, false); - let kept = l.imm(t, B::BitAnd, old, !(0xffffu64 << a.shift)); - l.put( - a.rd, - l.imm(t, B::BitOr, kept, (a.immediate as u64) << a.shift), - ); - } - } - Instruction::ArithImm(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, false); - let rhs = l.k(t, a.immediate as u64); - let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); - if a.flags { - l.put(a.rd, r); - l.store( - offset_of!(Cpu, flags), - l.flags(a.width, a.op, lhs, rhs, r, None), - ); - } else { - l.put_sp(a.rd, r) - } - } - Instruction::ArithReg(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let rhs = l.shift(a.width, l.reg(t, a.rm, true), a.shift, a.amount as u64); - let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); - if a.rd != 31 { - l.put(a.rd, r) - } - if a.flags { - l.store( - offset_of!(Cpu, flags), - l.flags(a.width, a.op, lhs, rhs, r, None), - ); - } - } - Instruction::ArithExt(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, false); - let extended = l.extend(l.reg(l.u64, a.rm, true), a.extend); - let rhs = l.cv(t, l.imm(l.u64, B::Shl, extended, a.amount as u64)); - let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); - if a.flags { - l.put(a.rd, r); - l.store( - offset_of!(Cpu, flags), - l.flags(a.width, a.op, lhs, rhs, r, None), - ); - } else { - l.put_sp(a.rd, r) - } - } - Instruction::LogicReg(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let shifted = l.shift(a.width, l.reg(t, a.rm, true), a.shift, a.amount as u64); - let rhs = if a.invert { - l.imm(t, B::BitXor, shifted, u64::MAX) - } else { - shifted - }; - let r = l.bin(t, logic_op(a.op), lhs, rhs); - l.put(a.rd, r); - if a.flags { - l.logic_flags(a.width, r) - } - } - Instruction::LogicImm(a) => { - let t = l.ty(a.width); - let r = l.imm(t, logic_op(a.op), l.reg(t, a.rn, true), a.immediate); - if a.flags { - l.put(a.rd, r); - l.logic_flags(a.width, r) - } else { - l.put_sp(a.rd, r) - } - } - Instruction::ArithCarry(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let loaded = l.reg(t, a.rm, true); - let rhs = if a.op == ArithOp::Sub { - l.imm(t, B::BitXor, loaded, u64::MAX) - } else { - loaded - }; - let carry = l.cv( - t, - l.imm( - l.u32, - B::BitAnd, - l.imm(l.u32, B::Shr, l.load(l.u32, offset_of!(Cpu, flags)), 29), - 1, - ), - ); - let partial = l.bin(t, B::Add, lhs, rhs); - let r = l.bin(t, B::Add, partial, carry); - if a.rd != 31 { - l.put(a.rd, r) - } - if a.flags { - let out = l.bin( - l.boolean, - B::BitOr, - l.cmp(C::Lt, partial, lhs), - l.cmp(C::Lt, r, partial), - ); - l.store( - offset_of!(Cpu, flags), - l.flags(a.width, ArithOp::Add, lhs, rhs, r, Some(out)), - ); - } - } - Instruction::CondCompare(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let rhs = match a.operand { - CompareOperand::Register(r) => l.reg(t, r, true), - CompareOperand::Immediate(n) => l.k(t, n as u64), - }; - let r = l.bin(t, arithmetic_op(a.op), lhs, rhs); - let flags = l.flags(a.width, a.op, lhs, rhs, r, None); - l.store( - offset_of!(Cpu, flags), - l.sel( - l.u32, - l.condition(a.cond), - flags, - l.k(l.u32, (a.nzcv as u64) << 28), - ), - ); - } - Instruction::Variable(a) => lower_variable(&l, a), - Instruction::Bitfield(a) => lower_bitfield(&l, a), - Instruction::Unary(a) => lower_unary(&l, a), - Instruction::Extract(a) => { - let t = l.ty(a.width); - let low = l.reg(t, a.rm, true); - let r = if a.lsb == 0 { - low - } else { - l.bin( - t, - B::BitOr, - l.imm(t, B::Shr, low, a.lsb as u64), - l.imm( - t, - B::Shl, - l.reg(t, a.rn, true), - a.width as u64 - a.lsb as u64, - ), - ) - }; - if a.rd != 31 { - l.put(a.rd, r) - } - } - Instruction::Mul(a) => { - let t = l.ty(a.width); - let p = l.bin(t, B::Mul, l.reg(t, a.rn, true), l.reg(t, a.rm, true)); - let r = l.bin(t, arithmetic_op(a.op), l.reg(t, a.ra, true), p); - if a.rd != 31 { - l.put(a.rd, r) - } - } - Instruction::MulLong(a) => { - let e = if a.signed { Extend::Sxtw } else { Extend::Uxtw }; - let p = l.bin( - l.u64, - B::Mul, - l.extend(l.reg(l.u64, a.rn, true), e), - l.extend(l.reg(l.u64, a.rm, true), e), - ); - let r = l.bin(l.u64, arithmetic_op(a.op), l.reg(l.u64, a.ra, true), p); - if a.rd != 31 { - l.put(a.rd, r) - } - } - Instruction::MulHigh(a) => lower_high(&l, a), - Instruction::Adr(a) => { - let here = if a.page { pc & 0xfffffffffffff000 } else { pc }; - l.put(a.rd, l.k(l.u64, here.wrapping_add(a.offset as u64))); - } - Instruction::Csel(a) => { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let rhs = l.reg(t, a.rm, true); - let alt = match a.opc { - 0 => rhs, - 1 => l.imm(t, B::Add, rhs, 1), - 2 => l.imm(t, B::BitXor, rhs, u64::MAX), - 3 => l.bin(t, B::Sub, l.k(t, 0), rhs), - _ => unreachable!(), - }; - if a.rd != 31 { - l.put(a.rd, l.sel(t, l.condition(a.cond), lhs, alt)); - } - } - Instruction::TestBranch(a) => { - let t = l.ty(a.width); - let v = l.reg(t, a.rt, true); - let subject = if let Some(bit) = a.bit_index { - l.imm(t, B::BitAnd, v, 1u64 << bit) - } else { - v - }; - l.branch( - pc, - a.offset, - l.cmp(if a.negate { C::Ne } else { C::Eq }, subject, l.k(t, 0)), - ); - } - Instruction::BCond(a) => l.branch(pc, a.offset, l.condition(a.cond)), - Instruction::B(offset) => l.ret(l.k(l.u64, pc.wrapping_add(offset as u64))), - Instruction::Call(a) => { - if a.link { - l.put(30, l.k(l.u64, pc.wrapping_add(4))) - } - l.ret(l.k(l.u64, pc.wrapping_add(a.target as u64))); - } - Instruction::Indirect(a) => { - let target = l.reg(l.u64, a.rn, false); - if a.link { - l.put(30, l.k(l.u64, pc.wrapping_add(4))) - } - l.ret(target); - } - Instruction::System(a) => lower_system(&l, a), - Instruction::Memory(a) => { - let trap = if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }; - if inline_memory { - let (address, post) = l.address_split(a.rn, a.addressing); - l.memory_access(address, a.size, a.op, a.rt, a.signed, post, |l| { - setup_memory(l, address, a.size, a.rt, a.signed, a.op); - if let Some((rn, value)) = post { - l.store(offset_of!(Cpu, writeback_value), value); - l.byte(offset_of!(Cpu, writeback_dest), rn as u64); - l.byte(offset_of!(Cpu, writeback), 1); - } - l.trap(pc.wrapping_add(4), trap); - }); - } else { - let address = l.address(a.rn, a.addressing); - setup_memory(&l, address, a.size, a.rt, a.signed, a.op); - l.trap(pc.wrapping_add(4), trap); - } - } - Instruction::Literal(a) => { - let address = l.k(l.u64, pc.wrapping_add(a.offset as u64)); - if inline_memory { - l.memory_access( - address, - a.size, - MemoryOp::Load, - a.rt, - SignExtend::None, - None, - |l| { - setup_memory( - l, - address, - a.size, - a.rt, - SignExtend::None, - MemoryOp::Load, - ); - l.trap(pc.wrapping_add(4), Trap::Load); - }, - ); - } else { - setup_memory(&l, address, a.size, a.rt, SignExtend::None, MemoryOp::Load); - l.trap(pc.wrapping_add(4), Trap::Load); - } - } - Instruction::SimdLiteral(a) => { - let address = l.k(l.u64, pc.wrapping_add(a.offset as u64)); - l.simd_trap_access(address, a.bytes as u64, a.rt, None, None, pc, Trap::Load); - } - Instruction::Exclusive(a) => { - setup_memory( - &l, - l.reg(l.u64, a.rn, false), - a.size, - a.rt, - SignExtend::None, - a.op, - ); - if a.op == MemoryOp::Store { - l.byte(offset_of!(Cpu, status_dest), a.rs as u64) - } - l.byte( - offset_of!(Cpu, exclusive), - if a.op == MemoryOp::Store { - CpuExclusive::Store as u64 - } else { - CpuExclusive::Load as u64 - }, - ); - l.trap( - pc.wrapping_add(4), - if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }, - ); - } - Instruction::Atomic(a) => { - // Read-modify-write needs a host atomic, so it always takes the machine - // slow path. `dests` receive the words read (31 discards) and `operands` - // are what the machine combines with memory (see `AtomicKind::apply`). - let (dests, operands): ([u8; 2], &[u8]) = match a.kind { - AtomicKind::Casp => ([a.rs, a.rs + 1], &[a.rs, a.rs + 1, a.rt, a.rt + 1]), - AtomicKind::Cas => ([a.rs, 31], &[a.rs, a.rt]), - AtomicKind::LoadPair => ([a.rt, a.rt2], &[]), - AtomicKind::StorePair => ([31, 31], &[a.rt, a.rt2]), - _ => ([a.rt, 31], &[a.rs]), - }; - l.store(offset_of!(Cpu, address), l.reg(l.u64, a.rn, false)); - l.byte(offset_of!(Cpu, width), a.size as u64); - l.byte(offset_of!(Cpu, dest), dests[0] as u64); - l.byte(offset_of!(Cpu, atomic_kind), a.kind as u64); - for (slot, &dest) in dests.iter().enumerate() { - l.byte(offset_of!(Cpu, atomic_dests) + slot, dest as u64); - } - for (slot, ®) in operands.iter().enumerate() { - // Register numbers past 30 read as XZR. - l.store( - offset_of!(Cpu, atomic_operands) + slot * 8, - l.reg(l.u64, reg, true), - ); - } - if a.kind == AtomicKind::StorePair { - l.byte(offset_of!(Cpu, status_dest), a.rs as u64); - } - l.trap(pc.wrapping_add(4), Trap::Atomic); - } - Instruction::SimdStruct(a) => { - use super::simd_struct::StructDesc; - // Element-wise accesses with lane merging need the machine slow path. - let d = a.desc; - let address = l.reg(l.u64, a.rn, false); - l.store(offset_of!(Cpu, address), address); - for (field, value) in [ - (offset_of!(StructDesc, store), u64::from(d.store)), - (offset_of!(StructDesc, shape), d.shape as u64), - (offset_of!(StructDesc, registers), u64::from(d.registers)), - ( - offset_of!(StructDesc, interleaved), - u64::from(d.interleaved), - ), - (offset_of!(StructDesc, esize), u64::from(d.esize)), - (offset_of!(StructDesc, lane), u64::from(d.lane)), - (offset_of!(StructDesc, q), u64::from(d.q)), - (offset_of!(StructDesc, rt), u64::from(d.rt)), - ] { - l.byte(offset_of!(Cpu, structure) + field, value); - } - let step = match a.post { - StructPost::None => None, - StructPost::Immediate => Some(l.k(l.u64, d.bytes())), - StructPost::Register(rm) => Some(l.reg(l.u64, rm, true)), - }; - if let Some(step) = step { - l.store( - offset_of!(Cpu, writeback_value), - l.bin(l.u64, B::Add, address, step), - ); - l.byte(offset_of!(Cpu, writeback_dest), a.rn as u64); - l.byte(offset_of!(Cpu, writeback), 1); - } - l.trap(pc.wrapping_add(4), Trap::SimdStruct); - } - Instruction::AddressTranslate(a) => { - l.store(offset_of!(Cpu, address), l.reg(l.u64, a.rt, true)); - l.store( - offset_of!(Cpu, value), - l.k(l.u64, u64::from(a.write) | u64::from(a.user) << 1), - ); - l.trap(pc.wrapping_add(4), Trap::AddressTranslate); - } - Instruction::Host(word) => { - // The host runs this word: see `host::run`. - l.store(offset_of!(Cpu, value), l.k(l.u64, word as u64)); - l.trap(pc.wrapping_add(4), Trap::Host); - } - Instruction::Pair(a) => { - let trap = if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }; - if inline_memory { - let (address, post) = l.address_split(a.rn, a.addressing); - l.pair_access(address, a, post, |l| { - setup_pair(l, address, a); - if let Some((rn, value)) = post { - l.store(offset_of!(Cpu, writeback_value), value); - l.byte(offset_of!(Cpu, writeback_dest), rn as u64); - l.byte(offset_of!(Cpu, writeback), 1); - } - l.trap(pc.wrapping_add(4), trap); - }); - } else { - let address = l.address(a.rn, a.addressing); - setup_pair(&l, address, a); - l.trap(pc.wrapping_add(4), trap); - } - } - Instruction::Clrex => l.byte(offset_of!(Cpu, monitor_valid), 0), - Instruction::Sev | Instruction::Sevl => l.byte(offset_of!(Cpu, event_set), 1), - Instruction::SimdImm(a) => { - let at = offset_of!(Cpu, v) + a.rd as usize * 16; - match a.combine { - ImmCombine::Set => { - l.store(at, l.k(l.u64, a.low)); - l.store(at + 8, l.k(l.u64, a.high)); - } - ImmCombine::Or | ImmCombine::AndNot => { - // A 64-bit form (`q == false`) also clears the upper half. - let (op, imm) = match a.combine { - ImmCombine::Or => (B::BitOr, a.low), - _ => (B::BitAnd, !a.low), - }; - l.store(at, l.imm(l.u64, op, l.load(l.u64, at), imm)); - l.store( - at + 8, - if a.q { - l.imm(l.u64, op, l.load(l.u64, at + 8), imm) - } else { - l.k(l.u64, 0) - }, - ); - } - } - } - Instruction::SimdMove { rd, rn } => { - let at_d = offset_of!(Cpu, v) + rd as usize * 16; - let at_n = offset_of!(Cpu, v) + rn as usize * 16; - let lo = l.load(l.u64, at_n); - let hi = l.load(l.u64, at_n + 8); - l.store(at_d, lo); - l.store(at_d + 8, hi); - } - Instruction::SimdInsert(a) => { - // Replace one lane of the destination with the low bits of a GPR. - let ebits = (1u64 << a.esize) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let per_dw = (64 / ebits) as usize; - let word = a.index as usize / per_dw; - let shift = (a.index as usize % per_dw) as u64 * ebits; - let old = l.load(l.u64, at_d + word * 8); - let cleared = l.bin(l.u64, B::BitAnd, old, l.k(l.u64, !(emask << shift))); - let value = l.bin( - l.u64, - B::BitAnd, - l.reg(l.u64, a.rn, true), - l.k(l.u64, emask), - ); - l.store( - at_d + word * 8, - l.bin(l.u64, B::BitOr, cleared, l.imm(l.u64, B::Shl, value, shift)), - ); - } - Instruction::SimdInsertElement(a) => { - // Copy one lane of Vn over one lane of Vd. The source is read before - // the store, so `rd == rn` is fine. - let ebits = (1u64 << a.esize) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let per_dw = (64 / ebits) as usize; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let (src_word, src_shift) = ( - a.src as usize / per_dw, - (a.src as usize % per_dw) as u64 * ebits, - ); - let (dst_word, dst_shift) = ( - a.dst as usize / per_dw, - (a.dst as usize % per_dw) as u64 * ebits, - ); - let source = l.load(l.u64, at_n + src_word * 8); - let value = l.imm( - l.u64, - B::BitAnd, - l.imm(l.u64, B::Shr, source, src_shift), - emask, - ); - let old = l.load(l.u64, at_d + dst_word * 8); - let cleared = l.imm(l.u64, B::BitAnd, old, !(emask << dst_shift)); - l.store( - at_d + dst_word * 8, - l.bin( - l.u64, - B::BitOr, - cleared, - l.imm(l.u64, B::Shl, value, dst_shift), - ), - ); - } - Instruction::SimdExtract(a) => { - // Move one lane into the low bits of a GPR, zero-extended. - let ebits = (1u64 << a.esize) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let per_dw = (64 / ebits) as usize; - let word = a.index as usize / per_dw; - let shift = (a.index as usize % per_dw) as u64 * ebits; - let source = l.load(l.u64, at_n + word * 8); - let value = l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, source, shift), emask); - l.put(a.rd, value); - } - Instruction::SimdDupElement(a) => { - let ebits = (1u64 << a.esize) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); - // Locate the requested element across the 128-bit source. - let lane = a.index as usize; - let lanes_per_dw = (64 / ebits) as usize; - let src = if lane / lanes_per_dw == 0 { - vn_lo - } else { - vn_hi - }; - let shift = (lane % lanes_per_dw) as u64 * ebits; - let element = l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, src, shift), emask); - // Replicate the element across the destination lanes. - let mut acc = l.k(l.u64, 0); - let lanes = (64 / ebits) as usize; - for lane in 0..lanes { - acc = l.bin( - l.u64, - B::BitOr, - acc, - l.imm(l.u64, B::Shl, element, lane as u64 * ebits), - ); - } - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - l.store(at_d, acc); - l.store(at_d + 8, if a.q { acc } else { l.k(l.u64, 0) }); - } - Instruction::SimdDup { rd, rn, esize, q } => { - let r = l.reg(l.u64, rn, true); - let val64 = match esize { - 0 => { - let b = l.imm(l.u64, B::BitAnd, r, 0xff); - l.bin(l.u64, B::Mul, b, l.k(l.u64, 0x0101_0101_0101_0101)) - } - 1 => { - let h = l.imm(l.u64, B::BitAnd, r, 0xffff); - l.bin(l.u64, B::Mul, h, l.k(l.u64, 0x0001_0001_0001_0001)) - } - 2 => { - let w = l.imm(l.u64, B::BitAnd, r, 0xffff_ffff); - l.bin(l.u64, B::BitOr, w, l.imm(l.u64, B::Shl, w, 32)) - } - _ => r, - }; - let at = offset_of!(Cpu, v) + rd as usize * 16; - l.store(at, val64); - let hi = if q { val64 } else { l.k(l.u64, 0) }; - l.store(at + 8, hi); - } - Instruction::SimdExt(a) => { - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); - let vm_lo = l.load(l.u64, at_m); - let vm_hi = l.load(l.u64, at_m + 8); - let (lo, hi) = if a.imm == 0 { - (vn_lo, vn_hi) - } else if a.imm == 8 { - (vn_hi, vm_lo) - } else if a.imm < 8 { - let s = a.imm as u64 * 8; - let r = (8 - a.imm as u64) * 8; - let lo = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::Shr, vn_lo, s), - l.imm(l.u64, B::Shl, vn_hi, r), - ); - let hi = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::Shr, vn_hi, s), - l.imm(l.u64, B::Shl, vm_lo, r), - ); - (lo, hi) - } else { - let shift = a.imm as u64 - 8; - let s = shift * 8; - let r = (8 - shift) * 8; - let lo = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::Shr, vn_hi, s), - l.imm(l.u64, B::Shl, vm_lo, r), - ); - let hi = l.bin( - l.u64, - B::BitOr, - l.imm(l.u64, B::Shr, vm_lo, s), - l.imm(l.u64, B::Shl, vm_hi, r), - ); - (lo, hi) - }; - l.store(at_d, lo); - l.store(at_d + 8, hi); - } - Instruction::SimdCompareZero(a) => { - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); - let ebits = (1u64 << a.size) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let halves = if a.q { 2 } else { 1 }; - for half in 0..2 { - if half >= halves { - l.store(at_d + half * 8, l.k(l.u64, 0)); - continue; - } - let v = if half == 0 { vn_lo } else { vn_hi }; - let mut acc = l.k(l.u64, 0); - let lanes = 8 >> a.size; - for lane in 0..lanes { - let shift = lane as u64 * ebits; - let field = l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, v, shift), emask); - let is_zero = l.cmp(C::Eq, field, l.k(l.u64, 0)); - let bits = l.sel(l.u64, is_zero, l.k(l.u64, emask), l.k(l.u64, 0)); - acc = l.bin(l.u64, B::BitOr, acc, l.imm(l.u64, B::Shl, bits, shift)); - } - l.store(at_d + half * 8, acc); - } - } - Instruction::SimdTable(a) => { - // Table bytes live in `len + 1` consecutive registers. Each index - // byte selects a table byte; out-of-range indices leave zero for - // `tbl` and keep the destination byte for `tbx`. - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let vm_lo = l.load(l.u64, at_m); - let vm_hi = l.load(l.u64, at_m + 8); - let regs = a.len as usize + 1; - let mut table = Vec::with_capacity(regs * 2); - for i in 0..regs { - let reg = (a.rn as usize + i) % 32; - let at = offset_of!(Cpu, v) + reg * 16; - table.push(l.load(l.u64, at)); - table.push(l.load(l.u64, at + 8)); - } - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let old_lo = if a.extend { - Some(l.load(l.u64, at_d)) - } else { - None - }; - let old_hi = if a.extend { - Some(l.load(l.u64, at_d + 8)) - } else { - None - }; - let halves = if a.q { 2 } else { 1 }; - let limit = (16 * regs) as u64; - for half in 0..2 { - if half >= halves { - l.store(at_d + half * 8, l.k(l.u64, 0)); - continue; - } - let indices = if half == 0 { vm_lo } else { vm_hi }; - let old = if half == 0 { old_lo } else { old_hi }; - let mut acc = l.k(l.u64, 0); - for byte in 0..8u64 { - let shift = byte * 8; - let index = - l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, indices, shift), 0xff); - let in_range = l.cmp(C::Lt, index, l.k(l.u64, limit)); - let mut value = l.k(l.u64, 0); - let sel = |c: Value, x: Value, y: Value| l.sel(l.u64, c, x, y); - for (b, entry) in table.iter().enumerate() { - let here = l.cmp( - C::Eq, - l.bin(l.u64, B::Shr, index, l.k(l.u64, 3)), - l.k(l.u64, b as u64), - ); - let byte_at = l.imm( - l.u64, - B::BitAnd, - l.bin( - l.u64, - B::Shr, - *entry, - l.bin( - l.u64, - B::Shl, - l.imm(l.u64, B::BitAnd, index, 7), - l.k(l.u64, 3), - ), - ), - 0xff, - ); - value = - sel(l.bin(l.boolean, B::BitAnd, in_range, here), byte_at, value); - } - if a.extend { - value = sel( - in_range, - value, - l.imm( - l.u64, - B::BitAnd, - l.imm(l.u64, B::Shr, old.unwrap(), shift), - 0xff, - ), - ); - } - acc = l.bin(l.u64, B::BitOr, acc, l.imm(l.u64, B::Shl, value, shift)); - } - l.store(at_d + half * 8, acc); - } - } - Instruction::SimdFmov(a) => { - // Pure bit move; no FP arithmetic involved. - if a.to_fp { - let v = l.reg(l.u64, a.rn, true); - let bits = if a.double { - v - } else { - l.imm(l.u64, B::BitAnd, v, 0xffff_ffff) - }; - let at = offset_of!(Cpu, v) + a.rd as usize * 16; - l.store(at, bits); - l.store(at + 8, l.k(l.u64, 0)); - } else { - let at = offset_of!(Cpu, v) + a.rn as usize * 16; - let v = l.load(l.u64, at); - let bits = if a.double { - v - } else { - l.imm(l.u64, B::BitAnd, v, 0xffff_ffff) - }; - l.put(a.rd, bits); - } - } - Instruction::SimdAlu(a) => { - use crate::aarch64::decode::SimdAluOp; - // Bitwise logical operations work on whole halves, lane width free. - if matches!( - a.op, - SimdAluOp::AddV | SimdAluOp::UMinV | SimdAluOp::UMaxV | SimdAluOp::UAddLV - ) { - // Horizontal add, minimum, maximum or widening add: reduce every - // lane of the source and store the result in lane 0, zeroing the - // rest. `UAddLV` keeps twice the lane width. - let ebits = (1u64 << a.size) * 8; - let mask_of = |bits: u64| { - if bits >= 64 { - u64::MAX - } else { - (1u64 << bits) - 1 - } - }; - let emask = mask_of(ebits); - let result_mask = if a.op == SimdAluOp::UAddLV { - mask_of(ebits * 2) - } else { - emask - }; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); - let halves = if a.q { 2 } else { 1 }; - let mut total = if a.op == SimdAluOp::UMinV { - l.k(l.u64, emask) - } else { - l.k(l.u64, 0) - }; - for half in 0..halves { - let v = if half == 0 { vn_lo } else { vn_hi }; - for lane in 0..(8 >> a.size) { - let shift = lane as u64 * ebits; - let field = - l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, v, shift), emask); - total = match a.op { - SimdAluOp::UMinV => { - l.sel(l.u64, l.cmp(C::Lt, field, total), field, total) - } - SimdAluOp::UMaxV => { - l.sel(l.u64, l.cmp(C::Gt, field, total), field, total) - } - _ => l.bin(l.u64, B::Add, total, field), - }; - } - } - l.store( - at_d, - l.bin(l.u64, B::BitAnd, total, l.k(l.u64, result_mask)), - ); - l.store(at_d + 8, l.k(l.u64, 0)); - } else if matches!( - a.op, - SimdAluOp::And - | SimdAluOp::Bic - | SimdAluOp::Orr - | SimdAluOp::Orn - | SimdAluOp::Eor - | SimdAluOp::Bsl - | SimdAluOp::Bit - | SimdAluOp::Bif - ) { - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - for half in 0..2 { - if half == 1 && !a.q { - l.store(at_d + 8, l.k(l.u64, 0)); - continue; - } - let x = l.load(l.u64, at_n + half * 8); - let y = l.load(l.u64, at_m + half * 8); - let not = |v| l.bin(l.u64, B::BitXor, v, l.k(l.u64, u64::MAX)); - let r = match a.op { - SimdAluOp::And => l.bin(l.u64, B::BitAnd, x, y), - SimdAluOp::Bic => l.bin(l.u64, B::BitAnd, x, not(y)), - SimdAluOp::Orr => l.bin(l.u64, B::BitOr, x, y), - SimdAluOp::Orn => l.bin(l.u64, B::BitOr, x, not(y)), - SimdAluOp::Eor => l.bin(l.u64, B::BitXor, x, y), - // The destination is both a source mask and the - // merge base for BSL/BIT/BIF. - _ => { - let d = l.load(l.u64, at_d + half * 8); - match a.op { - SimdAluOp::Bsl => l.bin( - l.u64, - B::BitOr, - l.bin(l.u64, B::BitAnd, x, d), - l.bin(l.u64, B::BitAnd, y, not(d)), - ), - _ => { - let selected = match a.op { - SimdAluOp::Bit => l.bin( - l.u64, - B::BitAnd, - l.bin(l.u64, B::BitXor, d, x), - y, - ), - _ => l.bin( - l.u64, - B::BitAnd, - l.bin(l.u64, B::BitXor, d, x), - not(y), - ), - }; - l.bin(l.u64, B::BitXor, d, selected) - } - } - } - }; - l.store(at_d + half * 8, r); - } - } else { - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let vn_lo = l.load(l.u64, at_n); - let vn_hi = l.load(l.u64, at_n + 8); - let vm_lo = l.load(l.u64, at_m); - let vm_hi = l.load(l.u64, at_m + 8); - let need_d = matches!(a.op, SimdAluOp::Mla | SimdAluOp::Mls); - // Pairwise forms consume adjacent lanes of the full source pair: - // lane 0/1 of Vn and Vm, then lane 2/3, and so on. - if matches!( - a.op, - SimdAluOp::UMaxP - | SimdAluOp::SMaxP - | SimdAluOp::UMinP - | SimdAluOp::SMinP - | SimdAluOp::AddP - ) { - let ebits = (1u64 << a.size) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let at_n = offset_of!(Cpu, v) + a.rn as usize * 16; - let at_m = offset_of!(Cpu, v) + a.rm as usize * 16; - let at_d = offset_of!(Cpu, v) + a.rd as usize * 16; - let s64 = l.signed(Width::X64); - let scmp = - |op: C, x: Value, y: Value| l.cmp(op, l.cv(s64, x), l.cv(s64, y)); - let ssel = |c: Value, x: Value, y: Value| l.sel(l.u64, c, x, y); - let sources = [ - l.load(l.u64, at_n), - l.load(l.u64, at_n + 8), - l.load(l.u64, at_m), - l.load(l.u64, at_m + 8), - ]; - // Each source is a 128-bit vector. Lane `2k` pairs with lane - // `2k + 1` and the result lands in lane `k`; the reduction of Vn - // fills the destination's low half and Vm its high half. - let per_source = (128 / ebits) as usize; - let reduced_lanes = per_source / 2; - let mut out = [l.k(l.u64, 0); 2]; - for (which, half) in [(0usize, 0usize), (2usize, 1usize)] { - let mut acc = l.k(l.u64, 0); - for k in 0..reduced_lanes { - let lanes_per_dw = (64 / ebits) as usize; - let read = |lane: usize| { - let double = lane / lanes_per_dw; - let shift = (lane % lanes_per_dw) as u64 * ebits; - l.imm( - l.u64, - B::BitAnd, - l.imm(l.u64, B::Shr, sources[which + double], shift), - emask, - ) - }; - let lo = read(2 * k); - let hi = read(2 * k + 1); - let r = match a.op { - SimdAluOp::AddP => l.bin(l.u64, B::Add, lo, hi), - SimdAluOp::UMaxP => ssel(l.cmp(C::Gt, lo, hi), lo, hi), - SimdAluOp::UMinP => ssel(l.cmp(C::Lt, lo, hi), lo, hi), - SimdAluOp::SMaxP => ssel(scmp(C::Gt, lo, hi), lo, hi), - _ => ssel(scmp(C::Lt, lo, hi), lo, hi), - }; - let masked = l.bin(l.u64, B::BitAnd, r, l.k(l.u64, emask)); - // per_source lanes reduce to exactly 64 bits, so the - // reduced lanes always pack into one doubleword. - acc = l.bin( - l.u64, - B::BitOr, - acc, - l.imm(l.u64, B::Shl, masked, k as u64 * ebits), - ); - } - out[half] = acc; - } - l.store(at_d, out[0]); - l.store(at_d + 8, if a.q { out[1] } else { l.k(l.u64, 0) }); - } else { - let vd_lo = if need_d { - Some(l.load(l.u64, at_d)) - } else { - None - }; - let vd_hi = if need_d { - Some(l.load(l.u64, at_d + 8)) - } else { - None - }; - let ebits = (1u64 << a.size) * 8; - let emask = if ebits == 64 { - u64::MAX - } else { - (1u64 << ebits) - 1 - }; - let s64 = l.signed(Width::X64); - let smax = u64::MAX >> (65 - ebits); - let smin = !smax; - let halves = if a.q { 2 } else { 1 }; - for half in 0..2 { - if half >= halves { - l.store(at_d + half * 8, l.k(l.u64, 0)); - continue; - } - let vn = if half == 0 { vn_lo } else { vn_hi }; - let vm = if half == 0 { vm_lo } else { vm_hi }; - let vd = if half == 0 { vd_lo } else { vd_hi }; - let mut acc = l.k(l.u64, 0); - let lanes = 8 >> a.size; - for lane in 0..lanes { - let shift = lane as u64 * ebits; - let fld = |v| { - l.imm(l.u64, B::BitAnd, l.imm(l.u64, B::Shr, v, shift), emask) - }; - let ext = |v: Value| { - let up = l.bin(s64, B::Shl, l.cv(s64, v), l.k(s64, 64 - ebits)); - l.imm(s64, B::Shr, up, 64 - ebits) - }; - let an = fld(vn); - let bn = fld(vm); - let sa = ext(an); - let sb = ext(bn); - let dn = vd.map(|v| fld(v)); - let scmp = |op: C, x: Value, y: Value| { - l.cmp(op, l.cv(s64, x), l.cv(s64, y)) - }; - let ssel = |c: Value, x: Value, y: Value| l.sel(l.u64, c, x, y); - let ssat = |v: Value| { - let over = scmp(C::Gt, v, l.k(l.u64, smax)); - let under = scmp(C::Lt, v, l.k(l.u64, smin)); - ssel(over, l.k(l.u64, smax), ssel(under, l.k(l.u64, smin), v)) - }; - let r = match a.op { - SimdAluOp::Add => l.bin(l.u64, B::Add, an, bn), - SimdAluOp::Sub => l.bin(l.u64, B::Sub, an, bn), - SimdAluOp::Mul => l.bin(l.u64, B::Mul, an, bn), - SimdAluOp::Mla => l.bin( - l.u64, - B::Add, - dn.unwrap(), - l.bin(l.u64, B::Mul, an, bn), - ), - SimdAluOp::Mls => l.bin( - l.u64, - B::Sub, - dn.unwrap(), - l.bin(l.u64, B::Mul, an, bn), - ), - SimdAluOp::CmEq => { - ssel(l.cmp(C::Eq, an, bn), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmGt => { - ssel(scmp(C::Gt, sa, sb), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmGe => { - ssel(scmp(C::Ge, sa, sb), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmHi => { - ssel(l.cmp(C::Gt, an, bn), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmHs => { - ssel(l.cmp(C::Ge, an, bn), l.k(l.u64, emask), l.k(l.u64, 0)) - } - SimdAluOp::CmTst => ssel( - l.cmp( - C::Ne, - l.bin(l.u64, B::BitAnd, an, bn), - l.k(l.u64, 0), - ), - l.k(l.u64, emask), - l.k(l.u64, 0), - ), - SimdAluOp::UMin => ssel(l.cmp(C::Lt, an, bn), an, bn), - SimdAluOp::UMax => ssel(l.cmp(C::Gt, an, bn), an, bn), - SimdAluOp::SMin => ssel(scmp(C::Lt, sa, sb), an, bn), - SimdAluOp::SMax => ssel(scmp(C::Gt, sa, sb), an, bn), - SimdAluOp::UAbd => { - let d = l.bin(l.u64, B::Sub, an, bn); - ssel( - l.cmp(C::Ge, an, bn), - d, - l.bin(l.u64, B::Sub, l.k(l.u64, 0), d), - ) - } - SimdAluOp::SAbd => { - let d = l.bin(s64, B::Sub, sa, sb); - let neg = l.bin(s64, B::Sub, l.k(s64, 0), d); - l.cv( - l.u64, - ssel( - scmp(C::Ge, sa, sb), - l.cv(l.u64, d), - l.cv(l.u64, neg), - ), - ) - } - SimdAluOp::SQAdd => { - ssat(l.cv(l.u64, l.bin(s64, B::Add, sa, sb))) - } - SimdAluOp::UQAdd => { - let s = l.bin(l.u64, B::Add, an, bn); - let carry = l.cmp(C::Lt, s, an); - let big = l.cmp(C::Gt, s, l.k(l.u64, emask)); - ssel( - l.bin(l.u64, B::BitOr, carry, big), - l.k(l.u64, emask), - s, - ) - } - SimdAluOp::SQSub => { - ssat(l.cv(l.u64, l.bin(s64, B::Sub, sa, sb))) - } - SimdAluOp::UQSub => { - let borrow = l.cmp(C::Gt, bn, an); - ssel(borrow, l.k(l.u64, 0), l.bin(l.u64, B::Sub, an, bn)) - } - SimdAluOp::SHAdd - | SimdAluOp::UHAdd - | SimdAluOp::SHSub - | SimdAluOp::UHSub - | SimdAluOp::SRHAdd - | SimdAluOp::URHAdd => { - let signed_op = matches!( - a.op, - SimdAluOp::SHAdd | SimdAluOp::SHSub | SimdAluOp::SRHAdd - ); - // Signed forms operate on sign-extended lanes, which - // keeps the halving shift meaningful for negatives. - let (op_a, op_b) = if signed_op { - (l.cv(l.u64, sa), l.cv(l.u64, sb)) - } else { - (an, bn) - }; - let shr = |v: Value| { - if signed_op { - l.cv(l.u64, l.imm(s64, B::Shr, l.cv(s64, v), 1)) - } else { - l.imm(l.u64, B::Shr, v, 1) - } - }; - let k = l.bin(l.u64, B::Sub, shr(op_a), shr(op_b)); - let k = if matches!( - a.op, - SimdAluOp::SHAdd - | SimdAluOp::UHAdd - | SimdAluOp::SRHAdd - | SimdAluOp::URHAdd - ) { - l.bin(l.u64, B::Add, shr(op_a), shr(op_b)) - } else { - k - }; - let a0 = l.bin(l.u64, B::BitAnd, op_a, l.k(l.u64, 1)); - let b0 = l.bin(l.u64, B::BitAnd, op_b, l.k(l.u64, 1)); - let corr = match a.op { - SimdAluOp::SHAdd | SimdAluOp::UHAdd => { - l.bin(l.u64, B::BitAnd, a0, b0) - } - SimdAluOp::SRHAdd | SimdAluOp::URHAdd => { - l.bin(l.u64, B::BitOr, a0, b0) - } - _ => l.bin( - l.u64, - B::BitAnd, - l.bin(l.u64, B::BitXor, a0, l.k(l.u64, 1)), - b0, - ), - }; - if matches!( - a.op, - SimdAluOp::SHAdd - | SimdAluOp::UHAdd - | SimdAluOp::SRHAdd - | SimdAluOp::URHAdd - ) { - l.bin(l.u64, B::Add, k, corr) - } else { - l.bin(l.u64, B::Sub, k, corr) - } - } - SimdAluOp::SSHl - | SimdAluOp::USHl - | SimdAluOp::SQShl - | SimdAluOp::UQShl - | SimdAluOp::SQRShl - | SimdAluOp::UQRShl => { - let signed_op = matches!( - a.op, - SimdAluOp::SSHl | SimdAluOp::SQShl | SimdAluOp::SQRShl - ); - let rounding = - matches!(a.op, SimdAluOp::SQRShl | SimdAluOp::UQRShl); - let saturating = matches!( - a.op, - SimdAluOp::SQShl - | SimdAluOp::UQShl - | SimdAluOp::SQRShl - | SimdAluOp::UQRShl - ); - // Shift amount: the lane's low byte, sign-extended to - // 64 bits (negative means a right shift). - let amt = l.cv( - s64, - l.bin( - s64, - B::Shl, - l.cv( - s64, - l.bin(l.u64, B::BitAnd, bn, l.k(l.u64, 0xff)), - ), - l.k(s64, 56), - ), - ); - let amt = l.cv(s64, l.bin(s64, B::Shr, amt, l.k(s64, 56))); - let left = scmp(C::Ge, amt, l.k(l.u64, 0)); - let n = ssel( - left, - l.cv(l.u64, amt), - l.cv(l.u64, l.bin(s64, B::Sub, l.k(s64, 0), amt)), - ); - let shl = |v: Value, k: Value| l.bin(l.u64, B::Shl, v, k); - let sar = |v: Value, k: Value| { - l.cv(l.u64, l.bin(s64, B::Shr, l.cv(s64, v), k)) - }; - let shr = |v: Value, k: Value| l.bin(l.u64, B::Shr, v, k); - // Beyond the lane width: a plain right shift keeps the - // sign fill (or zero), a rounding shift collapses to zero, - // and a saturating left shift pins to the lane bound. - let sign_fill = if signed_op { - ssel( - scmp(C::Lt, sa, l.k(l.u64, 0)), - l.k(l.u64, emask), - l.k(l.u64, 0), - ) - } else { - l.k(l.u64, 0) - }; - let sat_fill = if signed_op { - ssel( - scmp(C::Lt, sa, l.k(l.u64, 0)), - l.k(l.u64, smin), - l.k(l.u64, smax), - ) - } else { - l.k(l.u64, emask) - }; - let far = ssel( - left, - if saturating { sat_fill } else { l.k(l.u64, 0) }, - if rounding { l.k(l.u64, 0) } else { sign_fill }, - ); - // Shift when the amount stays inside the lane width. - // Clamping to `ebits - 1` keeps the host shift defined. - let kc = ssel( - l.cmp(C::Ge, n, l.k(l.u64, ebits)), - l.k(l.u64, ebits - 1), - n, - ); - let rnd = if rounding { - shl( - l.k(l.u64, 1), - l.bin(l.u64, B::Sub, kc, l.k(l.u64, 1)), - ) - } else { - l.k(l.u64, 0) - }; - let opnd = if signed_op { l.cv(l.u64, sa) } else { an }; - let t = if rounding { - // The rounding constant must not leak out of the lane. - l.bin( - l.u64, - B::BitAnd, - l.bin(l.u64, B::Add, opnd, rnd), - l.k(l.u64, emask), - ) - } else { - opnd - }; - let right = if rounding { - if signed_op { sar(t, kc) } else { shr(t, kc) } - } else if signed_op { - sar(opnd, kc) - } else { - shr(opnd, kc) - }; - let shifted = shl(opnd, kc); - let moved = ssel(left, shifted, right); - let moved = - ssel(l.cmp(C::Ge, n, l.k(l.u64, ebits)), far, moved); - if !saturating { - moved - } else if signed_op { - // Saturation only matters for left shifts; the - // right-shift result already fits the lane. Below - // the 64-bit lane the shifted value is exact, so the - // lane bounds test it directly; a 64-bit lane needs - // the round-trip check because the shift wraps. - let ov = if ebits == 64 { - l.cmp(C::Ne, sar(shifted, kc), opnd) - } else { - l.bin( - l.boolean, - B::BitOr, - scmp(C::Gt, shifted, l.k(l.u64, smax)), - scmp(C::Lt, shifted, l.k(l.u64, smin)), - ) - }; - let sat = ssel( - scmp(C::Lt, sa, l.k(l.u64, 0)), - l.k(l.u64, smin), - l.k(l.u64, smax), - ); - ssel(left, ssel(ov, sat, moved), moved) - } else { - let ov = if ebits == 64 { - l.cmp(C::Ne, shr(shifted, kc), an) - } else { - l.cmp(C::Gt, shifted, l.k(l.u64, emask)) - }; - ssel(left, ssel(ov, l.k(l.u64, emask), moved), moved) - } - } - _ => l.k(l.u64, 0), - }; - let masked = l.bin(l.u64, B::BitAnd, r, l.k(l.u64, emask)); - acc = l.bin( - l.u64, - B::BitOr, - acc, - l.imm(l.u64, B::Shl, masked, shift), - ); - } - l.store(at_d + half * 8, acc); - } - } - } - } - Instruction::SimdPair(a) => { - let trap = if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }; - let (address, post) = l.address_split(a.rn, a.addressing); - if !inline_memory { - l.simd_trap_access(address, a.bytes as u64, a.rt, Some(a.rt2), post, pc, trap); - } else { - let at_rt = offset_of!(Cpu, v) + a.rt as usize * 16; - let at_rt2 = offset_of!(Cpu, v) + a.rt2 as usize * 16; - let slow_block = l.new_block(); - let store = a.op == MemoryOp::Store; - let (bytes, chunk_count) = if a.bytes == 16 { (16, 4) } else { (8, 2) }; - let starts: [Value; 4] = [ - address, - l.imm(l.u64, B::Add, address, 8), - l.imm(l.u64, B::Add, address, 16), - l.imm(l.u64, B::Add, address, 24), - ]; - // stores: (reg_offset_base, half) per chunk - let src: [(usize, u64); 4] = if bytes == 16 { - [(at_rt, 0), (at_rt + 8, 0), (at_rt2, 0), (at_rt2 + 8, 0)] - } else { - [(at_rt, 0), (at_rt2, 0), (at_rt, 0), (at_rt, 0)] - }; - let mut hosts: [Option; 4] = [None; 4]; - for i in 0..chunk_count { - let next = l.new_block(); - let (hit, host) = l.dtlb_probe(starts[i], Size::Double, store); - l.branch_if(hit, next, slow_block); - l.block.set(next); - hosts[i] = Some(host); - } - if store { - for i in 0..chunk_count { - let host = hosts[i].unwrap(); - l.store_ptr(host, l.load(l.u64, src[i].0)); - } - } else { - for i in 0..chunk_count { - let host = hosts[i].unwrap(); - l.store(src[i].0, l.load_ptr(l.u64, host)); - } - // zero the untransferred half of each 128-bit register - if bytes == 8 { - l.store(at_rt + 8, l.k(l.u64, 0)); - l.store(at_rt2 + 8, l.k(l.u64, 0)); - } - } - if let Some((rn, value)) = post { - l.put_sp(rn, value); - } - // slow path, built last so every fast edge into it exists. - let resume = l.block.get(); - l.block.set(slow_block); - l.simd_trap_access(address, bytes as u64, a.rt, Some(a.rt2), post, pc, trap); - l.block.set(resume); - } - } - Instruction::SimdMemory(a) => { - let trap = if a.op == MemoryOp::Store { - Trap::Store - } else { - Trap::Load - }; - let (address, post) = l.address_split(a.rn, a.addressing); - if !inline_memory { - l.simd_trap_access(address, a.bytes as u64, a.rt, None, post, pc, trap); - } else { - let at_rt = offset_of!(Cpu, v) + a.rt as usize * 16; - let slow_block = l.new_block(); - let store = a.op == MemoryOp::Store; - // One chunk for widths up to 8 bytes, two 8-byte chunks for 16. - let wide = a.bytes == 16; - let size = match a.bytes { - 1 => Size::Byte, - 2 => Size::Half, - 4 => Size::Word, - _ => Size::Double, - }; - let chunk_count = if wide { 2 } else { 1 }; - let starts: [Value; 2] = [address, l.imm(l.u64, B::Add, address, 8)]; - let chunk_ty = l.uint((a.bytes.min(8) as u16) * 8); - let src: [usize; 2] = [at_rt, at_rt + 8]; - let mut hosts: [Option; 2] = [None; 2]; - for i in 0..chunk_count { - let next = l.new_block(); - let (hit, host) = l.dtlb_probe(starts[i], size, store); - l.branch_if(hit, next, slow_block); - l.block.set(next); - hosts[i] = Some(host); - } - if store { - for i in 0..chunk_count { - let host = hosts[i].unwrap(); - l.store_ptr(host, l.cv(chunk_ty, l.load(l.u64, src[i]))); - } - } else { - for i in 0..chunk_count { - let host = hosts[i].unwrap(); - l.store(src[i], l.cv(l.u64, l.load_ptr(chunk_ty, host))); - } - // Scalar loads zero the whole upper register, not just the - // untransferred half. - if a.bytes <= 8 { - l.store(at_rt + 8, l.k(l.u64, 0)); - if a.bytes < 8 { - let kept = (1u64 << (a.bytes * 8)) - 1; - let lo = l.imm(l.u64, B::BitAnd, l.load(l.u64, at_rt), kept); - l.store(at_rt, lo); - } - } - } - if let Some((rn, value)) = post { - l.put_sp(rn, value); - } - let resume = l.block.get(); - l.block.set(slow_block); - l.simd_trap_access(address, a.bytes as u64, a.rt, None, post, pc, trap); - l.block.set(resume); - } - } - Instruction::Nop - | Instruction::CacheOp - | Instruction::Dsb - | Instruction::Dmb - | Instruction::Yield => {} - Instruction::Wfe => l.trap(pc.wrapping_add(4), Trap::Wfe), - Instruction::Trap => l.trap(pc, Trap::Brk), - Instruction::Isb => l.trap(pc.wrapping_add(4), Trap::Isb), - Instruction::Tlbi => l.trap(pc.wrapping_add(4), Trap::Tlbi), - Instruction::DcZva(rt) => { - l.store(offset_of!(Cpu, address), l.reg(l.u64, rt, true)); - l.trap(pc.wrapping_add(4), Trap::DcZva) - } - Instruction::Eret => l.trap(pc.wrapping_add(4), Trap::Eret), - Instruction::Svc => l.trap(pc.wrapping_add(4), Trap::Svc), - Instruction::Psci => l.trap(pc.wrapping_add(4), Trap::Psci), - Instruction::Wfi => l.trap(pc.wrapping_add(4), Trap::Wfi), } pc = pc.wrapping_add(4); if ended { @@ -2165,531 +127,10 @@ pub fn compile_with(guest_pc: u64, bytes: &[u8], inline_memory: bool) -> Result< } } if !ended { - l.ret(l.k(l.u64, pc)) - } - Ok(Block::new(native::compile_owned(l.f.into_inner())?)) -} -/// Describe a GPR pair access to the machine: the first access in the usual -/// request fields, the second as `second_*`, completed after the first. -fn setup_pair(l: &Lower, address: Value, a: Pair) { - setup_memory(l, address, a.size, a.rt, a.signed, a.op); - l.store( - offset_of!(Cpu, second_address), - l.imm(l.u64, B::Add, address, a.size as u64), - ); - l.byte(offset_of!(Cpu, second_width), a.size as u64); - l.byte(offset_of!(Cpu, second_dest), a.rt2 as u64); - if a.signed != SignExtend::None { - l.byte(offset_of!(Cpu, second_signed), a.signed as u64) - } - if a.op == MemoryOp::Store { - l.store(offset_of!(Cpu, second_value), l.reg(l.u64, a.rt2, true)) - } - l.byte(offset_of!(Cpu, second_pending), 1); -} -fn setup_memory(l: &Lower, address: Value, size: Size, rt: u8, signed: SignExtend, op: MemoryOp) { - l.store(offset_of!(Cpu, address), address); - l.byte(offset_of!(Cpu, width), size as u64); - l.byte(offset_of!(Cpu, dest), rt as u64); - if signed != SignExtend::None { - l.byte(offset_of!(Cpu, load_signed), signed as u64) - } - if op == MemoryOp::Store { - l.store(offset_of!(Cpu, value), l.reg(l.u64, rt, true)) - } -} - -fn lower_variable(l: &Lower, a: Variable) { - let t = l.ty(a.width); - let lhs = l.reg(t, a.rn, true); - let rhs = l.reg(t, a.rm, true); - let bits = a.width as u64; - let r = match a.op { - VariableOp::Lsl | VariableOp::Lsr | VariableOp::Asr | VariableOp::Ror => { - let distance = l.imm(t, B::BitAnd, rhs, bits - 1); - match a.op { - VariableOp::Lsl => l.bin(t, B::Shl, lhs, distance), - VariableOp::Lsr => l.bin(t, B::Shr, lhs, distance), - VariableOp::Asr => { - let s = l.signed(a.width); - l.cv(t, l.bin(s, B::Shr, l.cv(s, lhs), l.cv(s, distance))) - } - VariableOp::Ror => { - let back = l.imm( - t, - B::BitAnd, - l.bin(t, B::Sub, l.k(t, bits), distance), - bits - 1, - ); - l.bin( - t, - B::BitOr, - l.bin(t, B::Shr, lhs, distance), - l.bin(t, B::Shl, lhs, back), - ) - } - _ => unreachable!(), - } - } - VariableOp::Udiv | VariableOp::Sdiv => { - let zero = l.k(t, 0); - let one = l.k(t, 1); - let by_zero = l.cmp(C::Eq, rhs, zero); - if a.op == VariableOp::Udiv { - let safe = l.sel(t, by_zero, one, rhs); - l.sel(t, by_zero, zero, l.bin(t, B::Div, lhs, safe)) - } else { - let minus_one = l.cmp(C::Eq, rhs, l.k(t, u64::MAX)); - let safe = l.sel(t, minus_one, one, l.sel(t, by_zero, one, rhs)); - let s = l.signed(a.width); - let quotient = l.cv(t, l.bin(s, B::Div, l.cv(s, lhs), l.cv(s, safe))); - let negated = l.bin(t, B::Sub, zero, lhs); - l.sel(t, by_zero, zero, l.sel(t, minus_one, negated, quotient)) - } - } - }; - if a.rd != 31 { - l.put(a.rd, r) - } -} -fn lower_bitfield(l: &Lower, a: Bitfield) { - let t = l.ty(a.width); - let bits = a.width as u64; - let len = a.len as u64; - let lsb = a.lsb as u64; - let source = l.reg(t, a.rn, true); - let mask = low_mask(a.len); - let width_mask = if a.width == Width::X64 { - u64::MAX - } else { - u32::MAX as u64 - }; - let extract = |signed| { - let raised = l.imm(t, B::Shl, source, bits - lsb - len); - l.shift( - a.width, - raised, - if signed { Shift::Asr } else { Shift::Lsr }, - bits - len, - ) - }; - let r = match a.op { - BitfieldOp::Unsigned => { - if a.extract { - extract(false) - } else { - l.imm( - t, - B::Shl, - l.imm(t, B::BitAnd, source, mask & width_mask), - lsb, - ) - } - } - BitfieldOp::Signed => { - if a.extract { - extract(true) - } else { - let raised = l.imm(t, B::Shl, source, bits - len); - let extended = l.shift(a.width, raised, Shift::Asr, bits - len); - l.imm(t, B::Shl, extended, lsb) - } - } - BitfieldOp::Insert => { - let value = if a.extract { - extract(false) - } else { - l.imm( - t, - B::Shl, - l.imm(t, B::BitAnd, source, mask & width_mask), - lsb, - ) - }; - let place = if a.extract { 0 } else { lsb }; - let hole = (mask << place) & width_mask; - let kept = l.imm(t, B::BitAnd, l.reg(t, a.rd, true), !hole & width_mask); - l.bin(t, B::BitOr, kept, value) - } - }; - if a.rd != 31 { - l.put(a.rd, r) - } -} -fn repeated(pattern: u64, unit: u32, bits: u32) -> u64 { - let mut r = 0; - let mut at = 0; - while at < bits { - r |= pattern << at; - at += unit - } - r -} -fn swap(l: &Lower, t: Type, v: Value, mask: u64, n: u64) -> Value { - let low = l.imm(t, B::BitAnd, v, mask); - let high = l.imm(t, B::BitAnd, l.imm(t, B::Shr, v, n), mask); - l.bin(t, B::BitOr, l.imm(t, B::Shl, low, n), high) -} -fn lower_unary(l: &Lower, a: Unary) { - let t = l.ty(a.width); - let bits = a.width as u32; - let mut v = l.reg(t, a.rn, true); - match a.op { - UnaryOp::Rev16 => v = swap(l, t, v, repeated(0xff, 16, bits), 8), - UnaryOp::Rev32 => { - v = swap(l, t, v, repeated(0xff, 16, bits), 8); - v = swap(l, t, v, repeated(0xffff, 32, bits), 16); - } - UnaryOp::Rev | UnaryOp::Rbit => { - if a.op == UnaryOp::Rbit { - for (mask, unit, n) in [(1, 2, 1), (3, 4, 2), (15, 8, 4)] { - v = swap(l, t, v, repeated(mask, unit, bits), n) - } - } - v = swap(l, t, v, repeated(0xff, 16, bits), 8); - v = swap(l, t, v, repeated(0xffff, 32, bits), 16); - if bits == 64 { - v = l.bin( - t, - B::BitOr, - l.imm(t, B::Shl, v, 32), - l.imm(t, B::Shr, v, 32), - ) - } - } - UnaryOp::Clz => { - let original = v; - let mut count = l.k(t, 0); - let mut step = bits / 2; - while step >= 1 { - let top = l.imm(t, B::Shr, v, (bits - step) as u64); - let empty = l.cmp(C::Eq, top, l.k(t, 0)); - v = l.sel(t, empty, l.imm(t, B::Shl, v, step as u64), v); - count = l.sel(t, empty, l.imm(t, B::Add, count, step as u64), count); - step /= 2; - } - v = l.sel( - t, - l.cmp(C::Eq, original, l.k(t, 0)), - l.k(t, bits as u64), - count, - ); - } - } - if a.rd != 31 { - l.put(a.rd, v) - } -} -fn lower_high(l: &Lower, a: MultiplyHigh) { - let t = l.u64; - let x = l.reg(t, a.rn, true); - let y = l.reg(t, a.rm, true); - let xl = l.imm(t, B::BitAnd, x, 0xffffffff); - let yl = l.imm(t, B::BitAnd, y, 0xffffffff); - let xh = l.imm(t, B::Shr, x, 32); - let yh = l.imm(t, B::Shr, y, 32); - let ll = l.bin(t, B::Mul, xl, yl); - let lh = l.bin(t, B::Mul, xl, yh); - let hl = l.bin(t, B::Mul, xh, yl); - let hh = l.bin(t, B::Mul, xh, yh); - let middle = l.bin( - t, - B::Add, - l.bin( - t, - B::Add, - l.imm(t, B::Shr, ll, 32), - l.imm(t, B::BitAnd, lh, 0xffffffff), - ), - l.imm(t, B::BitAnd, hl, 0xffffffff), - ); - let mut high = l.bin( - t, - B::Add, - l.bin(t, B::Add, hh, l.imm(t, B::Shr, lh, 32)), - l.bin( - t, - B::Add, - l.imm(t, B::Shr, hl, 32), - l.imm(t, B::Shr, middle, 32), - ), - ); - if a.signed { - let zero = l.k(t, 0); - let xn = l.bin(t, B::Sub, zero, l.imm(t, B::Shr, x, 63)); - let yn = l.bin(t, B::Sub, zero, l.imm(t, B::Shr, y, 63)); - high = l.bin(t, B::Sub, high, l.bin(t, B::BitAnd, y, xn)); - high = l.bin(t, B::Sub, high, l.bin(t, B::BitAnd, x, yn)); - } - if a.rd != 31 { - l.put(a.rd, high) - } -} - -fn system_offset(r: SystemRegister) -> Option { - use SystemRegister::*; - Some(match r { - SpsrEl1 => offset_of!(CpuSystem, spsr_el1), - ElrEl1 => offset_of!(CpuSystem, elr_el1), - EsrEl1 => offset_of!(CpuSystem, esr_el1), - FarEl1 => offset_of!(CpuSystem, far_el1), - ParEl1 => offset_of!(CpuSystem, par_el1), - IsrEl1 => offset_of!(CpuSystem, isr_el1), - IccSreEl1 => offset_of!(CpuSystem, icc_sre_el1), - IccPmrEl1 => offset_of!(CpuSystem, icc_pmr_el1), - IccBpr0El1 => offset_of!(CpuSystem, icc_bpr0_el1), - IccCtlrEl1 => offset_of!(CpuSystem, icc_ctlr_el1), - IccIgrpen0El1 => offset_of!(CpuSystem, icc_igrpen0_el1), - CntpCtlEl0 => offset_of!(CpuSystem, cntp_ctl_el0), - ActlrEl1 => offset_of!(CpuSystem, actlr_el1), - VbarEl1 => offset_of!(CpuSystem, vbar_el1), - CpacrEl1 => offset_of!(CpuSystem, cpacr_el1), - SctlrEl1 => offset_of!(CpuSystem, sctlr_el1), - Ttbr0El1 => offset_of!(CpuSystem, ttbr0_el1), - Ttbr1El1 => offset_of!(CpuSystem, ttbr1_el1), - TcrEl1 => offset_of!(CpuSystem, tcr_el1), - MairEl1 => offset_of!(CpuSystem, mair_el1), - TpidrEl0 => offset_of!(CpuSystem, tpidr_el0), - TpidrroEl0 => offset_of!(CpuSystem, tpidrro_el0), - CntvctEl0 => offset_of!(CpuSystem, cntvct_el0), - CntvCtlEl0 => offset_of!(CpuSystem, cntv_ctl_el0), - CntvCvalEl0 => offset_of!(CpuSystem, cntv_cval_el0), - TpidrEl1 => offset_of!(CpuSystem, tpidr_el1), - ContextidrEl1 => offset_of!(CpuSystem, contextidr_el1), - CsselrEl1 => offset_of!(CpuSystem, csselr_el1), - MdscrEl1 => offset_of!(CpuSystem, mdscr_el1), - CntkctlEl1 => offset_of!(CpuSystem, cntkctl_el1), - OslarEl1 => offset_of!(CpuSystem, oslar_el1), - OsdlrEl1 => offset_of!(CpuSystem, osdlr_el1), - Fpcr => offset_of!(CpuSystem, fpcr), - Fpsr => offset_of!(CpuSystem, fpsr), - Daif => offset_of!(CpuSystem, daif), - _ => return None, - }) -} -fn lower_system(l: &Lower, a: System) { - use SystemRegister::*; - l.flush_counter(); - let at = offset_of!(Cpu, system); - let level = match a.register { - SpEl0 => Some(0u64), - SpEl1 => Some(1), - SpEl2 => Some(2), - _ => None, - }; - if let Some(level) = level { - let current = l.cv(l.u64, l.load(l.u8, at + offset_of!(CpuSystem, el))); - let at_level = l.cmp(C::Eq, current, l.k(l.u64, level)); - let live = if level <= 1 { - let el1 = l.cmp(C::Eq, current, l.k(l.u64, 1)); - let selected = l.cmp( - if level == 0 { C::Eq } else { C::Ne }, - l.load(l.u8, at + offset_of!(CpuSystem, spsel)), - l.k(l.u8, 0), - ); - let el1_selected = l.bin(l.boolean, B::BitAnd, el1, selected); - if level == 0 { - l.bin(l.boolean, B::BitOr, at_level, el1_selected) - } else { - el1_selected - } - } else { - at_level - }; - let p = l.sel( - l.ptr, - live, - l.addr(Lower::reg_at(31)), - l.addr(at + offset_of!(CpuSystem, sp_el) + level as usize * 8), - ); - if a.read { - l.put(a.rt, l.load_ptr(l.u64, p)) - } else { - l.store_ptr(p, l.reg(l.u64, a.rt, true)) - } - return; - } - match a.register { - CurrentEl => l.put( - a.rt, - l.imm( - l.u64, - B::Shl, - l.cv(l.u64, l.load(l.u8, at + offset_of!(CpuSystem, el))), - 2, - ), - ), - Pan | PanSet => { - // PSTATE.PAN lives in bit 22 of the register form. - let pan = at + offset_of!(CpuSystem, pan); - if a.read { - l.put(a.rt, l.imm(l.u64, B::Shl, l.load(l.u64, pan), 22)); - } else if a.register == PanSet { - l.store(pan, l.k(l.u64, u64::from(a.immediate & 1))); - } else { - let bit = l.imm(l.u64, B::Shr, l.reg(l.u64, a.rt, true), 22); - l.store(pan, l.imm(l.u64, B::BitAnd, bit, 1)); - } - } - RazWi => { - if a.read { - l.put(a.rt, l.k(l.u64, 0)); - } - } - SpSel | SpSelSet => { - let selected = l.load(l.u8, at + offset_of!(CpuSystem, spsel)); - if a.read { - l.put(a.rt, l.cv(l.u64, selected)); - return; - } - let new = if a.register == SpSelSet { - l.k(l.u8, u64::from(a.immediate & 1)) - } else { - l.cv(l.u8, l.imm(l.u64, B::BitAnd, l.reg(l.u64, a.rt, true), 1)) - }; - // The live stack pointer belongs to bank `SP_EL1` when SPSel is 1 - // and `SP_EL0` when it is 0. Park it in its bank, then load the - // other one. Selecting the bank already in use is a store and a - // reload of the same slot. - let banks = at + offset_of!(CpuSystem, sp_el); - let bank = |select: Value| { - l.sel( - l.ptr, - l.cmp(C::Ne, select, l.k(l.u8, 0)), - l.addr(banks + 8), - l.addr(banks), - ) - }; - let live = l.load(l.u64, Lower::reg_at(31)); - l.store_ptr(bank(selected), live); - let incoming = l.load_ptr(l.u64, bank(new)); - l.store(Lower::reg_at(31), incoming); - l.store(at + offset_of!(CpuSystem, spsel), new); - } - DaifSet | DaifClear => { - let p = at + offset_of!(CpuSystem, daif); - let before = l.load(l.u64, p); - let updated = l.imm( - l.u64, - if a.register == DaifSet { - B::BitOr - } else { - B::BitAnd - }, - before, - if a.register == DaifSet { - a.immediate as u64 - } else { - (a.immediate as u64) ^ 0xf - }, - ); - l.store(p, updated); - } - Nzcv => { - if a.read { - l.put(a.rt, l.cv(l.u64, l.load(l.u32, offset_of!(Cpu, flags)))) - } else { - l.store( - offset_of!(Cpu, flags), - l.imm( - l.u32, - B::BitAnd, - l.cv(l.u32, l.reg(l.u64, a.rt, true)), - 0xf0000000, - ), - ) - } - } - OslsrEl1 => { - let either = l.bin( - l.u64, - B::BitOr, - l.load(l.u64, at + offset_of!(CpuSystem, oslar_el1)), - l.load(l.u64, at + offset_of!(CpuSystem, osdlr_el1)), - ); - let locked = l.cmp(C::Ne, either, l.k(l.u64, 0)); - l.put(a.rt, l.sel(l.u64, locked, l.k(l.u64, 2), l.k(l.u64, 0))); - } - CcsidrEl1 => { - // CSSELR bit 0 is InD, bits 3:1 the level - 1; L2 has no separate I-side. - use super::identification::{CCSIDR_L1D, CCSIDR_L1I, CCSIDR_L2}; - let select = l.imm( - l.u64, - B::BitAnd, - l.load(l.u64, at + offset_of!(CpuSystem, csselr_el1)), - 0xf, - ); - let is = |n: u64| l.cmp(C::Eq, select, l.k(l.u64, n)); - let l1_instruction = l.sel(l.u64, is(1), l.k(l.u64, CCSIDR_L1I), l.k(l.u64, CCSIDR_L2)); - l.put( - a.rt, - l.sel(l.u64, is(0), l.k(l.u64, CCSIDR_L1D), l1_instruction), - ); - } - CntvTvalEl0 => { - // The timer value is the signed 32-bit distance to the compare value: - // reads give the low 32 bits of `CVAL - count`, writes `CVAL = count + sext(value)`. - let counter = l.load(l.u64, at + offset_of!(CpuSystem, cntvct_el0)); - let compare = at + offset_of!(CpuSystem, cntv_cval_el0); - if a.read { - let distance = l.bin(l.u64, B::Sub, l.load(l.u64, compare), counter); - l.put(a.rt, l.imm(l.u64, B::BitAnd, distance, 0xffff_ffff)); - } else { - let delta = l.extend(l.reg(l.u64, a.rt, true), Extend::Sxtw); - l.store(compare, l.bin(l.u64, B::Add, counter, delta)); - } - } - Daif => { - let p = at + offset_of!(CpuSystem, daif); - if a.read { - l.put(a.rt, l.imm(l.u64, B::Shl, l.load(l.u64, p), 6)) - } else { - l.store( - p, - l.imm( - l.u64, - B::BitAnd, - l.imm(l.u64, B::Shr, l.reg(l.u64, a.rt, true), 6), - 0xf, - ), - ); - } - } - _ => { - if let Some(constant) = super::identification::value(a.register) { - l.put(a.rt, l.k(l.u64, constant)); - return; - } - let offset = system_offset(a.register).expect("decoded system register has storage"); - let p = at + offset; - if a.read { - let v = if a.register == CntvCtlEl0 { - let counter = l.load(l.u64, at + offset_of!(CpuSystem, cntvct_el0)); - let compare = l.load(l.u64, at + offset_of!(CpuSystem, cntv_cval_el0)); - let reached = l.cmp(C::Ge, counter, compare); - l.bin( - l.u64, - B::BitOr, - l.load(l.u64, p), - l.sel(l.u64, reached, l.k(l.u64, 4), l.k(l.u64, 0)), - ) - } else { - l.load(l.u64, p) - }; - l.put(a.rt, v); - } else { - let from = l.reg(l.u64, a.rt, true); - l.store( - p, - if a.register == CntvCtlEl0 { - l.imm(l.u64, B::BitAnd, from, 3) - } else { - from - }, - ); - } - } + lifter.exit(pc); } + let (function, _) = lifter.into_parts(); + Ok(Block::new(native::compile_owned(function)?)) } #[cfg(test)] @@ -2713,20 +154,6 @@ mod tests { assert_eq!(cpu.flags, 0x80000000); } #[test] - fn zero_division_and_signed_overflow_do_not_trap_host() { - // UDIV X2,X0,X1; SDIV X3,X0,X1. - let block = compile(0, &bytes(&[0x9ac10802, 0x9ac10c03])).unwrap(); - let mut cpu = Cpu::default(); - cpu.x[0] = 1 << 63; - cpu.x[1] = 0; - block.run(&mut cpu); - assert_eq!(cpu.x[2], 0); - assert_eq!(cpu.x[3], 0); - cpu.x[1] = u64::MAX; - block.run(&mut cpu); - assert_eq!(cpu.x[3], 1 << 63); - } - #[test] fn pair_post_index_records_deferred_access_and_writeback() { // LDP X0,X1,[X2],#16. let block = compile(0x8000, &bytes(&[0xa8c10440])).unwrap(); @@ -2937,282 +364,4 @@ mod tests { run_inline(&mut cpu2, &[0xf9400001, SVC]); assert_eq!(cpu2.trap, Trap::Load); } - #[test] - fn simd_compare_zero_masks_each_lane() { - // cmeq v1.16b, v0.16b, #0. - let block = compile(0x1000, &bytes(&[0x4e209801])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[0][0] = 0x00ff_0000_0000_00ff; - cpu.v[0][1] = 0xffff_ffff_ffff_ffff; - block.run(&mut cpu); - assert_eq!(cpu.v[1][0], 0xff00_ffff_ffff_ff00); - assert_eq!(cpu.v[1][1], 0); - // Half-width form operates on v0.8b and clears the top half. - let block = compile(0x1000, &bytes(&[0x0e209801])).unwrap(); - cpu.v[0][0] = 0x00ff_0000_0000_00ff; - cpu.v[0][1] = 0x1234_5678_9abc_def0; - cpu.v[1] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; - block.run(&mut cpu); - assert_eq!(cpu.v[1][0], 0xff00_ffff_ffff_ff00); - assert_eq!(cpu.v[1][1], 0); - } - #[test] - fn simd_fmov_moves_bits_between_gpr_and_fpr() { - // fmov x2, d2 then fmov d3, x2 round-trips the low doubleword. - let block = compile(0x1000, &bytes(&[0x9e660042, 0x9e670043])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[2] = [0x1122_3344_5566_7788, 0xdead_beef_dead_beef]; - block.run(&mut cpu); - assert_eq!(cpu.x[2], 0x1122_3344_5566_7788); - assert_eq!(cpu.v[3], [0x1122_3344_5566_7788, 0]); - // 32-bit form truncates to the low word. - let block = compile(0x1000, &bytes(&[0x1e260022])).unwrap(); - cpu.v[1] = [0xaaaa_bbbb_cccc_dddd, 0]; - block.run(&mut cpu); - assert_eq!(cpu.x[2] as u32, 0xcccc_dddd); - } - #[test] - fn simd_pairwise_reductions_match_lane_layout() { - let setup = || { - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; - cpu.v[2] = [0x1112_1314_1516_1718, 0x191a_1b1c_1d1e_1f20]; - cpu - }; - for (word, want) in [ - ( - 0x6e22a420u32, - [0x0a0c_0e10_0204_0608u64, 0x1a1c_1e20_1214_1618], - ), - (0x6e22ac20, [0x090b_0d0f_0103_0507, 0x191b_1d1f_1113_1517]), - (0x4e22bc20, [0x1317_1b1f_0307_0b0f, 0x3337_3b3f_2327_2b2f]), - (0x4ea2bc20, [0x1618_1a1c_0608_0a0c, 0x3638_3a3c_2628_2a2c]), - (0x4ee2bc20, [0x0a0c_0e10_1214_1618, 0x2a2c_2e30_3234_3638]), - ] { - let mut cpu = setup(); - let block = compile(0x1000, &bytes(&[word])).unwrap(); - block.run(&mut cpu); - assert_eq!(cpu.v[0], want, "{word:08x}"); - } - } - #[test] - fn simd_horizontal_add_sums_lanes() { - for (word, lanes, want) in [ - (0x4e31b820u32, 16usize, 0x88u64), - (0x4e71b820, 8, 0x4048), - (0x4eb1b820, 4, 0x1c20_2428), - ] { - let block = compile(0x1000, &bytes(&[word])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; - block.run(&mut cpu); - let _ = lanes; - assert_eq!(cpu.v[0], [want, 0], "{word:08x}"); - } - // The 64-bit form reads only the low half of the source. - let block = compile(0x1000, &bytes(&[0x0e31b820])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; - block.run(&mut cpu); - assert_eq!(cpu.v[0], [0x24, 0]); - } - #[test] - fn simd_dup_from_element_replicates_a_lane() { - for (word, want) in [ - // dup v0.4h, v1.h[0] -> element 0x0708 in every halfword. - (0x0e020420u32, [0x0708_0708_0708_0708u64, 0]), - // dup v0.8h, v1.h[7] -> element 0x090a, both halves. - (0x4e1e0420, [0x090a_090a_090a_090a, 0x090a_090a_090a_090a]), - // dup v0.4s, v1.s[1] -> element 0x0102_0304. - (0x4e0c0420, [0x0102_0304_0102_0304, 0x0102_0304_0102_0304]), - // dup v0.2d, v1.d[1] -> element 0x090a_0b0c_0d0e_0f10. - (0x4e180420, [0x090a_0b0c_0d0e_0f10, 0x090a_0b0c_0d0e_0f10]), - // dup v0.16b, v1.b[3] -> element 0x05. - (0x4e070420, [0x0505_0505_0505_0505, 0x0505_0505_0505_0505]), - ] { - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; - let block = compile(0x1000, &bytes(&[word])).unwrap(); - block.run(&mut cpu); - assert_eq!(cpu.v[0], want, "{word:08x}"); - } - } - #[test] - fn simd_table_lookup_selects_bytes() { - // tbl v0.16b, {v1.16b, v2.16b}, v3.16b with indices 0,2,4,... - let block = compile(0x1000, &bytes(&[0x4e032020])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; - cpu.v[2] = [0x1716_1514_1312_1110, 0x1f1e_1d1c_1b1a_1918]; - cpu.v[3] = [0x0604_0200_0e0c_0a08, 0x1614_1210_1e1c_1a18]; - block.run(&mut cpu); - assert_eq!(cpu.v[0], [0x0604_0200_0e0c_0a08, 0x1614_1210_1e1c_1a18]); - // Out-of-range indices yield zero for tbl. - let block = compile(0x1000, &bytes(&[0x4e020020])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; - cpu.v[2] = [0xff01_ff00_ff00_ff00, 0xffff_ffff_ffff_ffff]; - block.run(&mut cpu); - assert_eq!(cpu.v[0], [0x0001_0000_0000_0000, 0]); - // tbx keeps the destination byte where the index is out of range. - let block = compile(0x1000, &bytes(&[0x4e021020])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; - cpu.v[2] = [0xff01_ff00_ff00_ff00, 0xffff_ffff_ffff_ffff]; - cpu.v[0] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; - block.run(&mut cpu); - assert_eq!(cpu.v[0], [0xde01_be00_de00_be00, 0xdead_beef_dead_beef]); - } - #[test] - fn simd_alu_lane_semantics() { - // Ground truth computed with the host's own byte arithmetic. - struct Case { - word: u32, - a: [u64; 2], - b: [u64; 2], - want: [u64; 2], - } - let cases = [ - // add v0.16b, v1.16b, v2.16b wraps per byte - Case { - word: 0x4e228420, - a: [0x0102_03ff_0405_0607, 0x0f10_1120_2122_2324], - b: [0x0101_0101_0101_0101, 0x0102_0304_0506_0708], - want: [0x0203_0400_0506_0708, 0x1012_1424_2628_2a2c], - }, - // sub v0.16b, v1.16b, v2.16b - Case { - word: 0x6e228420, - a: [0x0001_0203_0405_0607, 0], - b: [0x0101_0101_0101_0101, 0], - want: [0xff00_0102_0304_0506, 0], - }, - // cmhs v0.16b, v1.16b, v2.16b (unsigned >=) - Case { - word: 0x6e223c20, - a: [0x00ff_0080_7f7f_8000, 0], - b: [0x0001_0180_7f80_8000, 0], - want: [0xffff_00ff_ff00_ffff, 0xffff_ffff_ffff_ffff], - }, - // umin v0.8h, v1.8h, v2.8h - Case { - word: 0x6e626c20, - a: [0x0003_0002_0001_0000, 0], - b: [0x0002_0003_0000_0001, 0], - want: [0x0002_0002_0000_0000, 0], - }, - // uqadd v0.8h, v1.8h, v2.8h saturates - Case { - word: 0x6e620c20, - a: [0xffff_8000_0001_0000, 0], - b: [0x0001_9000_0002_0001, 0], - want: [0xffff_ffff_0003_0001, 0], - }, - // shadd v0.16b, v1.16b, v2.16b rounds down (arithmetic) - Case { - word: 0x4e220420, - a: [0x0303_fefe_0505_00ff, 0], - b: [0x0303_fefe_0505_00ff, 0], - want: [0x0303_fefe_0505_00ff, 0], - }, - // sshl v0.8h by signed amounts in the low byte of each lane - Case { - word: 0x4e624420, - a: [0x0004_0003_0002_0001, 0], - b: [0x0000_0002_00ff_0001, 0], - want: [0x0004_000c_0001_0002, 0], - }, - // mla/mls fold in the destination *lane*, not the whole half - Case { - word: 0x4e229420, - a: [0x0202_0202_0202_0202, 0], - b: [0x0505_0505_0505_0505, 0], - want: [0x0a0a_0a0a_0a0a_0a0a, 0], - }, - Case { - word: 0x6e229420, - a: [0x0202_0202_0202_0202, 0], - b: [0x0505_0505_0505_0505, 0], - want: [0xf6f6_f6f6_f6f6_f6f6, 0], - }, - // bsl: destination starts zero, so the result is v2 unchanged - Case { - word: 0x6e621c20, - a: [0xaaaa_aaaa_aaaa_aaaa, 0], - b: [0x5555_5555_5555_5555, 0], - want: [0x5555_5555_5555_5555, 0], - }, - // bit with a zero destination inserts the masked v1 bits - Case { - word: 0x6ea21c20, - a: [0x0f0f_0f0f_0f0f_0f0f, 0], - b: [0xaaaa_aaaa_aaaa_aaaa, 0], - want: [0x0a0a_0a0a_0a0a_0a0a, 0], - }, - // bif with a zero destination inserts the inverse-masked bits - Case { - word: 0x6ee21c20, - a: [0x0f0f_0f0f_0f0f_0f0f, 0], - b: [0xaaaa_aaaa_aaaa_aaaa, 0], - want: [0x0505_0505_0505_0505, 0], - }, - // shadd of two -1 bytes stays -1 (arithmetic halving) - Case { - word: 0x4e220420, - a: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], - b: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], - want: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], - }, - // shsub of 0 - 1 halves to -1 - Case { - word: 0x4e222420, - a: [0, 0], - b: [0x0101_0101_0101_0101, 0], - want: [0xffff_ffff_ffff_ffff, 0], - }, - // srhadd of 1 and 2 rounds to 2 - Case { - word: 0x4e221420, - a: [0x0101_0101_0101_0101, 0], - b: [0x0202_0202_0202_0202, 0], - want: [0x0202_0202_0202_0202, 0], - }, - // sshl v0.8h: lane 0 shifts left by 1, lane 1 right by 1 - Case { - word: 0x4e624420, - a: [0x0008_0004_0002_0001, 0], - b: [0x0000_00ff_0001_0001, 0], - want: [0x0008_0002_0004_0002, 0], - }, - // uqshl v0.8h saturates an overflow to the lane maximum - Case { - word: 0x6e624c20, - a: [0x0000_0000_0000_8000, 0], - b: [0x0000_0000_0000_0001, 0], - want: [0x0000_0000_0000_ffff, 0], - }, - // sqshl v0.8h saturates a positive overflow to the lane maximum - Case { - word: 0x4e624c20, - a: [0x0000_0000_0000_4000, 0], - b: [0x0000_0000_0000_0001, 0], - want: [0x0000_0000_0000_7fff, 0], - }, - // sqrshl v0.8h rounds before shifting right - Case { - word: 0x4e625c20, - a: [0x0000_0000_0000_0003, 0], - b: [0x0000_0000_0000_00ff, 0], - want: [0x0000_0000_0000_0002, 0], - }, - ]; - for case in cases { - let block = compile(0x1000, &bytes(&[case.word])).unwrap(); - let mut cpu = Cpu::default(); - cpu.v[1] = case.a; - cpu.v[2] = case.b; - block.run(&mut cpu); - assert_eq!(cpu.v[0], case.want, "{:08x}", case.word); - } - } } diff --git a/src/emulator/src/aarch64/cpu.rs b/src/emulator/src/aarch64/cpu.rs index 8e37c64..42aee5f 100644 --- a/src/emulator/src/aarch64/cpu.rs +++ b/src/emulator/src/aarch64/cpu.rs @@ -1,5 +1,5 @@ -use super::decode::{AtomicKind, SignExtend}; -use super::simd_struct::StructDesc; +use volt_isa_aarch64::decode::{AtomicKind, SignExtend}; +use volt_isa_aarch64::simd_struct::StructDesc; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] @@ -153,8 +153,27 @@ pub struct Cpu { pub v: [[u64; 2]; 32], /// Inline data TLB for generated loads and stores. Filled by the machine. pub dtlb: Dtlb, + /// Apple IMP-DEF registers of the `op1` 0 to 6 banks the SPTM monitor configures. It + /// is the last field so growing it never moves the fields generated code reaches + /// with short immediate offsets (`v`, `dtlb`). + pub apple_bank: ImpDefBank, } +/// `S3__c15` registers, indexed by `volt_isa_aarch64::decode::apple_bank_slot`. +#[derive(Clone, Debug)] +pub struct ImpDefBank(pub [u64; volt_isa_aarch64::decode::APPLE_BANK_SLOTS]); + +impl Default for ImpDefBank { + fn default() -> Self { + Self([0; volt_isa_aarch64::decode::APPLE_BANK_SLOTS]) + } +} + +/// `HCR_EL2.TGE` (bit 27): exceptions taken from EL0 route to EL2. +pub const HCR_TGE: u64 = 1 << 27; +/// `HCR_EL2.E2H` (bit 34): VHE. EL1 register names accessed from EL2 reach the EL2 banks. +pub const HCR_E2H: u64 = 1 << 34; + #[derive(Clone, Debug)] #[repr(C)] pub struct System { @@ -197,8 +216,36 @@ pub struct System { pub icc_bpr0_el1: u64, pub icc_ctlr_el1: u64, pub icc_igrpen0_el1: u64, + pub icc_igrpen1_el1: u64, + pub icc_bpr1_el1: u64, pub cntp_ctl_el0: u64, + pub cntp_cval_el0: u64, pub actlr_el1: u64, + /// EL2 banks (SPTM runs at EL2, GL2). `SP_EL2` is `sp_el[2]`. + pub sctlr_el2: u64, + pub tcr_el2: u64, + pub ttbr0_el2: u64, + pub ttbr1_el2: u64, + pub mair_el2: u64, + pub vbar_el2: u64, + pub elr_el2: u64, + pub spsr_el2: u64, + pub esr_el2: u64, + pub far_el2: u64, + pub hcr_el2: u64, + pub tpidr_el2: u64, + pub cntvoff_el2: u64, + /// Apple IMP-DEF configuration registers, indexed by `SystemRegister::Apple`. + pub apple: [u64; volt_isa_aarch64::decode::APPLE_SLOTS], + /// `MIDR_EL1`, `ID_AA64PFR0_EL1` and `ID_AA64MMFR0_EL1`: the identity this CPU + /// advertises, set by the board. + pub midr_el1: u64, + pub id_aa64pfr0_el1: u64, + pub id_aa64mmfr0_el1: u64, + /// PAuth keys, low word first: APIA, APIB, APDA, APDB, APGA. + pub pac_keys: [u64; 10], + /// `ID_AA64ISAR1_EL1`: which pointer authentication algorithms the CPU advertises. + pub id_aa64isar1_el1: u64, } impl Default for System { fn default() -> Self { @@ -217,6 +264,19 @@ impl Default for System { ttbr0_el1: 0, ttbr1_el1: 0, tcr_el1: 0, + sctlr_el2: 0, + tcr_el2: 0, + ttbr0_el2: 0, + ttbr1_el2: 0, + mair_el2: 0, + vbar_el2: 0, + elr_el2: 0, + spsr_el2: 0, + esr_el2: 0, + far_el2: 0, + hcr_el2: 0, + tpidr_el2: 0, + cntvoff_el2: 0, mair_el1: 0, tpidr_el0: 0, tpidrro_el0: 0, @@ -241,8 +301,17 @@ impl Default for System { // PRIbits 7: eight priority bits. icc_ctlr_el1: 7 << 8, icc_igrpen0_el1: 0, + icc_igrpen1_el1: 0, + icc_bpr1_el1: 0, cntp_ctl_el0: 0, + cntp_cval_el0: 0, actlr_el1: 0, + apple: [0; volt_isa_aarch64::decode::APPLE_SLOTS], + midr_el1: crate::aarch64::identification::DEFAULT_MIDR_EL1, + id_aa64pfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64PFR0_EL1, + id_aa64mmfr0_el1: crate::aarch64::identification::DEFAULT_ID_AA64MMFR0_EL1, + pac_keys: [0; 10], + id_aa64isar1_el1: crate::aarch64::identification::DEFAULT_ID_AA64ISAR1_EL1, } } } diff --git a/src/emulator/src/aarch64/environment.rs b/src/emulator/src/aarch64/environment.rs new file mode 100644 index 0000000..f87ee18 --- /dev/null +++ b/src/emulator/src/aarch64/environment.rs @@ -0,0 +1,1043 @@ +//! The emulator's [`Environment`]: guest state is the [`Cpu`] struct, memory goes +//! through the inline data TLB and the trap-and-resume slow path, and system +//! instructions read and write [`CpuSystem`]. +use super::cpu::{Cpu, DTLB_ENTRIES, Exclusive as CpuExclusive, System as CpuSystem, Trap}; +use core::mem::offset_of; +use volt_ir::function::{BinOp as B, CmpOp as C, Value}; +use volt_isa_aarch64::decode::*; +use volt_lift_aarch64::{Builder, Environment, LiftError, MemoryAccess, Writeback}; + +/// Lowers guest instructions against [`Cpu`]: the lifter's machine policy. +pub(super) struct CpuEnvironment { + /// Plain loads and stores get an inline data-TLB fast path and do not end + /// the block; the block scan must use `host::ends_block(_, true)`. + inline_memory: bool, + /// Guest instructions started but not yet added to `cntvct_el0`. The + /// counter is bumped once per exit path (`exit`) and before anything that + /// reads it, instead of with a load/add/store per instruction. + pending: u64, +} +impl CpuEnvironment { + pub(super) fn new(inline_memory: bool) -> Self { + Self { + inline_memory, + pending: 0, + } + } + /// A word the host runs (see `host::run`) instead of the lifter: it + /// counts as a retired instruction, hands the raw word to the machine, and + /// ends the block. + pub(super) fn host(&mut self, b: &Builder, pc: u64, word: u32) { + self.pending += 1; + b.store(offset_of!(Cpu, value), b.k(b.i64, word as u64)); + self.trap(b, pc.wrapping_add(4), Trap::Host); + } + fn add_counter(&self, b: &Builder, n: u64) { + let at = offset_of!(Cpu, system) + offset_of!(CpuSystem, cntvct_el0); + b.store(at, b.imm(b.i64, B::Add, b.load(b.i64, at), n)); + } + /// Make `cntvct_el0` in memory current on the path being emitted. + fn flush_counter(&mut self, b: &Builder) { + let n = core::mem::replace(&mut self.pending, 0); + if n != 0 { + self.add_counter(b, n); + } + } + /// Leave the function with the counter current. `pending` is not cleared: + /// sibling paths (slow blocks) share the main path's count. + fn leave(&self, b: &Builder, next_pc: Value) { + if self.pending != 0 { + self.add_counter(b, self.pending); + } + b.set_ret(next_pc) + } + /// Hand the base-register update of an access to the machine, which applies + /// it once the access has completed. + fn defer_writeback(&self, b: &Builder, w: Writeback) { + b.store(offset_of!(Cpu, writeback_value), w.value); + b.byte(offset_of!(Cpu, writeback_dest), w.register as u64); + b.byte(offset_of!(Cpu, writeback), 1); + } + + /// Probe the inline data TLB for `address`. Returns `(hit, host_pointer)`; + /// the pointer is only meaningful when `hit`. A hit requires the page to be + /// cached for this access kind and the access to be naturally aligned (so it + /// cannot cross a page), because the tag compare covers the low `size - 1` + /// address bits too. + fn dtlb_probe(&self, b: &Builder, address: Value, size: Size, write: bool) -> (Value, Value) { + let n = size as u64; + // Entry offset = ((address >> 12) & (ENTRIES - 1)) * 32 = (address >> 7) & mask. + let index = b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, address, 7), + (DTLB_ENTRIES as u64 - 1) << 5, + ); + let entry = b.bin(b.ptr, B::Add, b.state(), index); + // Constant displacements fold into the load addressing. + let field = |at: usize| { + b.load_ptr( + b.i64, + b.imm(b.ptr, B::Add, entry, (offset_of!(Cpu, dtlb) + at) as u64), + ) + }; + let tag = field(if write { 8 } else { 0 }); + let addend = field(16); + let host = b.bin(b.ptr, B::Add, address, addend); + let want = b.imm(b.i64, B::BitAnd, address, !0xfffu64 | (n - 1)); + // Last, so the backend can fuse it into the branch that consumes it. + let hit = b.cmp(C::Eq, tag, want); + (hit, host) + } + fn fast_load(&self, b: &Builder, host: Value, size: Size, rt: u8, signed: SignExtend) { + let bits = size as u16 * 8; + let raw = b.load_ptr(b.int(bits), host); + if rt == 31 { + return; + } + let mut v = b.cv(b.i64, raw); + if signed != SignExtend::None && bits < 64 { + let n = 64 - bits as u64; + let extended = b.shift(Width::X64, b.imm(b.i64, B::Shl, v, n), Shift::Asr, n); + v = if signed == SignExtend::To32 { + b.imm(b.i64, B::BitAnd, extended, 0xffff_ffff) + } else { + extended + }; + } + b.put(rt, v); + } + fn fast_store(&self, b: &Builder, host: Value, size: Size, rt: u8) { + let t = b.int(size as u16 * 8); + b.store_ptr(host, b.cv(t, b.reg(b.i64, rt, true))); + } + /// Emit a load or store with an inline fast path. On a data-TLB hit the + /// access happens in place and emission continues in a fresh block; on a + /// miss control goes to a slow block that hands the access to the machine + /// (`slow` fills in the request) and returns. + fn memory_access( + &self, + b: &Builder, + address: Value, + size: Size, + op: MemoryOp, + rt: u8, + signed: SignExtend, + post: Option, + slow: impl FnOnce(&Self), + ) { + let write = op == MemoryOp::Store; + let (hit, host) = self.dtlb_probe(b, address, size, write); + let (fast_block, slow_block) = (b.new_block(), b.new_block()); + b.branch_if(hit, fast_block, slow_block); + b.set_block(slow_block); + slow(self); + b.set_block(fast_block); + if write { + self.fast_store(b, host, size, rt); + } else { + self.fast_load(b, host, size, rt, signed); + } + if let Some(w) = post { + b.put_sp(w.register, w.value); + } + } + /// Inline fast path for a GPR load/store pair: one data-TLB probe covers + /// both accesses when the first is naturally aligned and the second starts + /// in the same page. Anything else takes `slow`, which hands the whole pair + /// to the machine. + fn pair_access( + &self, + b: &Builder, + address: Value, + a: Pair, + post: Option, + slow: impl FnOnce(&Self), + ) { + let write = a.op == MemoryOp::Store; + let n = a.size as u64; + let (hit, host) = self.dtlb_probe(b, address, a.size, write); + let (check_block, fast_block, slow_block) = (b.new_block(), b.new_block(), b.new_block()); + b.branch_if(hit, check_block, slow_block); + // `address` is aligned to `n`, so the second access leaves the page + // exactly when it starts on a page boundary. + b.set_block(check_block); + let second_at = b.imm(b.i64, B::Add, address, n); + let offset = b.imm(b.i64, B::BitAnd, second_at, 0xfff); + let same_page = b.cmp(C::Ne, offset, b.k(b.i64, 0)); + b.branch_if(same_page, fast_block, slow_block); + b.set_block(slow_block); + slow(self); + b.set_block(fast_block); + let second = b.imm(b.ptr, B::Add, host, n); + if write { + self.fast_store(b, host, a.size, a.rt); + self.fast_store(b, second, a.size, a.rt2); + } else { + self.fast_load(b, host, a.size, a.rt, a.signed); + self.fast_load(b, second, a.size, a.rt2, a.signed); + } + if let Some(w) = post { + b.put_sp(w.register, w.value); + } + } + fn trap(&self, b: &Builder, pc: u64, t: Trap) { + b.byte(offset_of!(Cpu, trap), t as u64); + self.leave(b, b.k(b.i64, pc)) + } + + /// Slow path for a whole SIMD vector or vector-pair access: describe the + /// full instruction in the CPU request fields (`vector_dest`, optionally + /// `second_*`) and trap out; the machine completes every byte and resumes + /// after the instruction. + fn simd_trap_access( + &self, + b: &Builder, + address: Value, + bytes: u64, + rt: u8, + rt2: Option, + post: Option, + pc: u64, + trap: Trap, + ) { + b.store(offset_of!(Cpu, address), address); + b.byte(offset_of!(Cpu, width), bytes); + b.byte(offset_of!(Cpu, dest), rt as u64); + b.byte(offset_of!(Cpu, vector_dest), 1); + if let Some(rt2) = rt2 { + let second = b.imm(b.i64, B::Add, address, if bytes == 16 { 16 } else { 8 }); + b.store(offset_of!(Cpu, second_address), second); + b.byte(offset_of!(Cpu, second_width), bytes); + b.byte(offset_of!(Cpu, second_dest), rt2 as u64); + b.byte(offset_of!(Cpu, second_vector), 1); + b.byte(offset_of!(Cpu, second_pending), 1); + } + if let Some(w) = post { + self.defer_writeback(b, w); + } + self.trap(b, pc.wrapping_add(4), trap); + } + /// Describe a GPR pair access to the machine: the first access in the usual + /// request fields, the second as `second_*`, completed after the first. + fn setup_pair(&self, b: &Builder, address: Value, a: Pair) { + self.setup_memory(b, address, a.size, a.rt, a.signed, a.op); + b.store( + offset_of!(Cpu, second_address), + b.imm(b.i64, B::Add, address, a.size as u64), + ); + b.byte(offset_of!(Cpu, second_width), a.size as u64); + b.byte(offset_of!(Cpu, second_dest), a.rt2 as u64); + if a.signed != SignExtend::None { + b.byte(offset_of!(Cpu, second_signed), a.signed as u64) + } + if a.op == MemoryOp::Store { + b.store(offset_of!(Cpu, second_value), b.reg(b.i64, a.rt2, true)) + } + b.byte(offset_of!(Cpu, second_pending), 1); + } + fn setup_memory( + &self, + b: &Builder, + address: Value, + size: Size, + rt: u8, + signed: SignExtend, + op: MemoryOp, + ) { + b.store(offset_of!(Cpu, address), address); + b.byte(offset_of!(Cpu, width), size as u64); + b.byte(offset_of!(Cpu, dest), rt as u64); + if signed != SignExtend::None { + b.byte(offset_of!(Cpu, load_signed), signed as u64) + } + if op == MemoryOp::Store { + b.store(offset_of!(Cpu, value), b.reg(b.i64, rt, true)) + } + } + + /// Apply a base-register update the way a non-inline access does: a + /// pre-index update immediately, a post-index one via the machine. + fn writeback_eager(&self, b: &Builder, post: Option) { + match post { + Some(w) if w.pre_index => b.put_sp(w.register, w.value), + Some(w) => self.defer_writeback(b, w), + None => {} + } + } + // `a.ordered` (`ldar`, `stlr`, `ldapr`, ...) and the unscaled/unprivileged offset forms + // run as the plain access: the machines model no memory ordering or EL0 permission check. + fn scalar(&self, b: &Builder, pc: u64, a: Memory, address: Value, post: Option) { + let trap = if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }; + if self.inline_memory { + self.memory_access(b, address, a.size, a.op, a.rt, a.signed, post, |env| { + env.setup_memory(b, address, a.size, a.rt, a.signed, a.op); + if let Some(w) = post { + env.defer_writeback(b, w); + } + env.trap(b, pc.wrapping_add(4), trap); + }); + } else { + self.writeback_eager(b, post); + self.setup_memory(b, address, a.size, a.rt, a.signed, a.op); + self.trap(b, pc.wrapping_add(4), trap); + } + } + fn literal(&self, b: &Builder, pc: u64, a: Literal, address: Value) { + if self.inline_memory { + self.memory_access( + b, + address, + a.size, + MemoryOp::Load, + a.rt, + a.signed, + None, + |env| { + env.setup_memory(b, address, a.size, a.rt, a.signed, MemoryOp::Load); + env.trap(b, pc.wrapping_add(4), Trap::Load); + }, + ); + } else { + self.setup_memory(b, address, a.size, a.rt, a.signed, MemoryOp::Load); + self.trap(b, pc.wrapping_add(4), Trap::Load); + } + } + fn pair(&self, b: &Builder, pc: u64, a: Pair, address: Value, post: Option) { + let trap = if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }; + if self.inline_memory { + self.pair_access(b, address, a, post, |env| { + env.setup_pair(b, address, a); + if let Some(w) = post { + env.defer_writeback(b, w); + } + env.trap(b, pc.wrapping_add(4), trap); + }); + } else { + self.writeback_eager(b, post); + self.setup_pair(b, address, a); + self.trap(b, pc.wrapping_add(4), trap); + } + } + fn simd_pair( + &self, + b: &Builder, + pc: u64, + a: SimdPair, + address: Value, + post: Option, + ) { + let trap = if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }; + if !self.inline_memory { + self.simd_trap_access( + b, + address, + a.bytes as u64, + a.rt, + Some(a.rt2), + post, + pc, + trap, + ); + } else { + let at_rt = b.v_at(a.rt); + let at_rt2 = b.v_at(a.rt2); + let slow_block = b.new_block(); + let store = a.op == MemoryOp::Store; + let (bytes, chunk_count) = if a.bytes == 16 { (16, 4) } else { (8, 2) }; + let starts: [Value; 4] = [ + address, + b.imm(b.i64, B::Add, address, 8), + b.imm(b.i64, B::Add, address, 16), + b.imm(b.i64, B::Add, address, 24), + ]; + // stores: (reg_offset_base, half) per chunk + let src: [(usize, u64); 4] = if bytes == 16 { + [(at_rt, 0), (at_rt + 8, 0), (at_rt2, 0), (at_rt2 + 8, 0)] + } else { + [(at_rt, 0), (at_rt2, 0), (at_rt, 0), (at_rt, 0)] + }; + let mut hosts: [Option; 4] = [None; 4]; + for i in 0..chunk_count { + let next = b.new_block(); + let (hit, host) = self.dtlb_probe(b, starts[i], Size::Double, store); + b.branch_if(hit, next, slow_block); + b.set_block(next); + hosts[i] = Some(host); + } + if store { + for i in 0..chunk_count { + let host = hosts[i].unwrap(); + b.store_ptr(host, b.load(b.i64, src[i].0)); + } + } else { + for i in 0..chunk_count { + let host = hosts[i].unwrap(); + b.store(src[i].0, b.load_ptr(b.i64, host)); + } + // zero the untransferred half of each 128-bit register + if bytes == 8 { + b.store(at_rt + 8, b.k(b.i64, 0)); + b.store(at_rt2 + 8, b.k(b.i64, 0)); + } + } + if let Some(w) = post { + b.put_sp(w.register, w.value); + } + // slow path, built last so every fast edge into it exists. + let resume = b.block(); + b.set_block(slow_block); + self.simd_trap_access(b, address, bytes as u64, a.rt, Some(a.rt2), post, pc, trap); + b.set_block(resume); + } + } + // `a.access` (`ldr q` vs one-register `ld1`) and unscaled offsets move the same bytes. + fn simd_memory( + &self, + b: &Builder, + pc: u64, + a: SimdMemory, + address: Value, + post: Option, + ) { + let trap = if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }; + if !self.inline_memory { + self.simd_trap_access(b, address, a.bytes as u64, a.rt, None, post, pc, trap); + } else { + let at_rt = b.v_at(a.rt); + let slow_block = b.new_block(); + let store = a.op == MemoryOp::Store; + // One chunk for widths up to 8 bytes, two 8-byte chunks for 16. + let wide = a.bytes == 16; + let size = match a.bytes { + 1 => Size::Byte, + 2 => Size::Half, + 4 => Size::Word, + _ => Size::Double, + }; + let chunk_count = if wide { 2 } else { 1 }; + let starts: [Value; 2] = [address, b.imm(b.i64, B::Add, address, 8)]; + let chunk_ty = b.int((a.bytes.min(8) as u16) * 8); + let src: [usize; 2] = [at_rt, at_rt + 8]; + let mut hosts: [Option; 2] = [None; 2]; + for i in 0..chunk_count { + let next = b.new_block(); + let (hit, host) = self.dtlb_probe(b, starts[i], size, store); + b.branch_if(hit, next, slow_block); + b.set_block(next); + hosts[i] = Some(host); + } + if store { + for i in 0..chunk_count { + let host = hosts[i].unwrap(); + b.store_ptr(host, b.cv(chunk_ty, b.load(b.i64, src[i]))); + } + } else { + for i in 0..chunk_count { + let host = hosts[i].unwrap(); + b.store(src[i], b.cv(b.i64, b.load_ptr(chunk_ty, host))); + } + // Scalar loads zero the whole upper register, not just the + // untransferred half. + if a.bytes <= 8 { + b.store(at_rt + 8, b.k(b.i64, 0)); + if a.bytes < 8 { + let kept = (1u64 << (a.bytes * 8)) - 1; + let lo = b.imm(b.i64, B::BitAnd, b.load(b.i64, at_rt), kept); + b.store(at_rt, lo); + } + } + } + if let Some(w) = post { + b.put_sp(w.register, w.value); + } + let resume = b.block(); + b.set_block(slow_block); + self.simd_trap_access(b, address, a.bytes as u64, a.rt, None, post, pc, trap); + b.set_block(resume); + } + } + // `a.order` (`ldaxr`/`stlxr`) is ignored: the monitor and host accesses already order. + fn exclusive(&self, b: &Builder, pc: u64, a: Exclusive) { + self.setup_memory( + b, + b.reg(b.i64, a.rn, false), + a.size, + a.rt, + SignExtend::None, + a.op, + ); + if a.op == MemoryOp::Store { + b.byte(offset_of!(Cpu, status_dest), a.rs as u64) + } + b.byte( + offset_of!(Cpu, exclusive), + if a.op == MemoryOp::Store { + CpuExclusive::Store as u64 + } else { + CpuExclusive::Load as u64 + }, + ); + self.trap( + b, + pc.wrapping_add(4), + if a.op == MemoryOp::Store { + Trap::Store + } else { + Trap::Load + }, + ); + } + // `a.order` (acquire/release) is ignored: every host read-modify-write is sequentially + // consistent. + fn atomic(&self, b: &Builder, pc: u64, a: Atomic) { + // Read-modify-write needs a host atomic, so it always takes the machine + // slow path. `dests` receive the words read (31 discards) and `operands` + // are what the machine combines with memory (see `AtomicKind::apply`). + let (dests, operands): ([u8; 2], &[u8]) = match a.kind { + AtomicKind::Casp => ([a.rs, a.rs + 1], &[a.rs, a.rs + 1, a.rt, a.rt + 1]), + AtomicKind::Cas => ([a.rs, 31], &[a.rs, a.rt]), + AtomicKind::LoadPair => ([a.rt, a.rt2], &[]), + AtomicKind::StorePair => ([31, 31], &[a.rt, a.rt2]), + _ => ([a.rt, 31], &[a.rs]), + }; + b.store(offset_of!(Cpu, address), b.reg(b.i64, a.rn, false)); + b.byte(offset_of!(Cpu, width), a.size as u64); + b.byte(offset_of!(Cpu, dest), dests[0] as u64); + b.byte(offset_of!(Cpu, atomic_kind), a.kind as u64); + for (slot, &dest) in dests.iter().enumerate() { + b.byte(offset_of!(Cpu, atomic_dests) + slot, dest as u64); + } + for (slot, ®) in operands.iter().enumerate() { + // Register numbers past 30 read as XZR. + b.store( + offset_of!(Cpu, atomic_operands) + slot * 8, + b.reg(b.i64, reg, true), + ); + } + if a.kind == AtomicKind::StorePair { + b.byte(offset_of!(Cpu, status_dest), a.rs as u64); + } + self.trap(b, pc.wrapping_add(4), Trap::Atomic); + } + fn simd_struct(&self, b: &Builder, pc: u64, a: SimdStruct) { + use volt_isa_aarch64::simd_struct::StructDesc; + // Element-wise accesses with lane merging need the machine slow path. + let d = a.desc; + let address = b.reg(b.i64, a.rn, false); + b.store(offset_of!(Cpu, address), address); + for (field, value) in [ + (offset_of!(StructDesc, store), u64::from(d.store)), + (offset_of!(StructDesc, shape), d.shape as u64), + (offset_of!(StructDesc, registers), u64::from(d.registers)), + ( + offset_of!(StructDesc, interleaved), + u64::from(d.interleaved), + ), + (offset_of!(StructDesc, esize), u64::from(d.esize)), + (offset_of!(StructDesc, lane), u64::from(d.lane)), + (offset_of!(StructDesc, q), u64::from(d.q)), + (offset_of!(StructDesc, rt), u64::from(d.rt)), + ] { + b.byte(offset_of!(Cpu, structure) + field, value); + } + let step = match a.post { + StructPost::None => None, + StructPost::Immediate => Some(b.k(b.i64, d.bytes())), + StructPost::Register(rm) => Some(b.reg(b.i64, rm, true)), + }; + if let Some(step) = step { + b.store( + offset_of!(Cpu, writeback_value), + b.bin(b.i64, B::Add, address, step), + ); + b.byte(offset_of!(Cpu, writeback_dest), a.rn as u64); + b.byte(offset_of!(Cpu, writeback), 1); + } + self.trap(b, pc.wrapping_add(4), Trap::SimdStruct); + } + + fn lower_system(&mut self, b: &Builder, a: System) { + use SystemRegister::*; + self.flush_counter(b); + let at = offset_of!(Cpu, system); + let level = match a.register { + SpEl0 => Some(0u64), + SpEl1 => Some(1), + SpEl2 => Some(2), + _ => None, + }; + if let Some(level) = level { + let current = b.cv(b.i64, b.load(b.i8, at + offset_of!(CpuSystem, el))); + let at_level = b.cmp(C::Eq, current, b.k(b.i64, level)); + let live = if level <= 1 { + let el1 = b.cmp(C::Eq, current, b.k(b.i64, 1)); + let selected = b.cmp( + if level == 0 { C::Eq } else { C::Ne }, + b.load(b.i8, at + offset_of!(CpuSystem, spsel)), + b.k(b.i8, 0), + ); + let el1_selected = b.bin(b.boolean, B::BitAnd, el1, selected); + if level == 0 { + b.bin(b.boolean, B::BitOr, at_level, el1_selected) + } else { + el1_selected + } + } else { + at_level + }; + let p = b.sel( + b.ptr, + live, + b.addr(b.reg_at(31)), + b.addr(at + offset_of!(CpuSystem, sp_el) + level as usize * 8), + ); + if a.read { + b.put(a.rt, b.load_ptr(b.i64, p)) + } else { + b.store_ptr(p, b.reg(b.i64, a.rt, true)) + } + return; + } + if let Some((el1, el2)) = vhe_banks(a.register) { + // HCR_EL2.E2H: EL1 register names accessed from EL2 reach the EL2 banks. + let current = b.cv(b.i64, b.load(b.i8, at + offset_of!(CpuSystem, el))); + let el2_mode = b.cmp(C::Eq, current, b.k(b.i64, 2)); + let e2h = b.cmp( + C::Ne, + b.imm( + b.i64, + B::BitAnd, + b.load(b.i64, at + offset_of!(CpuSystem, hcr_el2)), + super::cpu::HCR_E2H, + ), + b.k(b.i64, 0), + ); + let redirect = b.bin(b.boolean, B::BitAnd, el2_mode, e2h); + let p = b.sel(b.ptr, redirect, b.addr(at + el2), b.addr(at + el1)); + if a.read { + b.put(a.rt, b.load_ptr(b.i64, p)); + } else { + b.store_ptr(p, b.reg(b.i64, a.rt, true)); + } + return; + } + match a.register { + CurrentEl => b.put( + a.rt, + b.imm( + b.i64, + B::Shl, + b.cv(b.i64, b.load(b.i8, at + offset_of!(CpuSystem, el))), + 2, + ), + ), + Pan | PanSet => { + // PSTATE.PAN lives in bit 22 of the register form. + let pan = at + offset_of!(CpuSystem, pan); + if a.read { + b.put(a.rt, b.imm(b.i64, B::Shl, b.load(b.i64, pan), 22)); + } else if a.register == PanSet { + b.store(pan, b.k(b.i64, u64::from(a.immediate & 1))); + } else { + let bit = b.imm(b.i64, B::Shr, b.reg(b.i64, a.rt, true), 22); + b.store(pan, b.imm(b.i64, B::BitAnd, bit, 1)); + } + } + RazWi { .. } | AppleZero { .. } => { + if a.read { + b.put(a.rt, b.k(b.i64, 0)); + } + } + SpSel | SpSelSet => { + let selected = b.load(b.i8, at + offset_of!(CpuSystem, spsel)); + if a.read { + b.put(a.rt, b.cv(b.i64, selected)); + return; + } + let new = if a.register == SpSelSet { + b.k(b.i8, u64::from(a.immediate & 1)) + } else { + b.cv(b.i8, b.imm(b.i64, B::BitAnd, b.reg(b.i64, a.rt, true), 1)) + }; + // The live stack pointer belongs to bank `SP_EL1` when SPSel is 1 + // and `SP_EL0` when it is 0. Park it in its bank, then load the + // other one. Selecting the bank already in use is a store and a + // reload of the same slot. + let banks = at + offset_of!(CpuSystem, sp_el); + let bank = |select: Value| { + b.sel( + b.ptr, + b.cmp(C::Ne, select, b.k(b.i8, 0)), + b.addr(banks + 8), + b.addr(banks), + ) + }; + let live = b.load(b.i64, b.reg_at(31)); + b.store_ptr(bank(selected), live); + let incoming = b.load_ptr(b.i64, bank(new)); + b.store(b.reg_at(31), incoming); + b.store(at + offset_of!(CpuSystem, spsel), new); + } + DaifSet | DaifClear => { + let p = at + offset_of!(CpuSystem, daif); + let before = b.load(b.i64, p); + let updated = b.imm( + b.i64, + if a.register == DaifSet { + B::BitOr + } else { + B::BitAnd + }, + before, + if a.register == DaifSet { + a.immediate as u64 + } else { + (a.immediate as u64) ^ 0xf + }, + ); + b.store(p, updated); + } + Nzcv => { + if a.read { + b.put(a.rt, b.cv(b.i64, b.load(b.i32, offset_of!(Cpu, flags)))) + } else { + b.store( + offset_of!(Cpu, flags), + b.imm( + b.i32, + B::BitAnd, + b.cv(b.i32, b.reg(b.i64, a.rt, true)), + 0xf0000000, + ), + ) + } + } + OslsrEl1 => { + let either = b.bin( + b.i64, + B::BitOr, + b.load(b.i64, at + offset_of!(CpuSystem, oslar_el1)), + b.load(b.i64, at + offset_of!(CpuSystem, osdlr_el1)), + ); + let locked = b.cmp(C::Ne, either, b.k(b.i64, 0)); + b.put(a.rt, b.sel(b.i64, locked, b.k(b.i64, 2), b.k(b.i64, 0))); + } + CcsidrEl1 => { + // CSSELR bit 0 is InD, bits 3:1 the level - 1; L2 has no separate I-side. + use super::identification::{CCSIDR_L1D, CCSIDR_L1I, CCSIDR_L2}; + let select = b.imm( + b.i64, + B::BitAnd, + b.load(b.i64, at + offset_of!(CpuSystem, csselr_el1)), + 0xf, + ); + let is = |n: u64| b.cmp(C::Eq, select, b.k(b.i64, n)); + let l1_instruction = + b.sel(b.i64, is(1), b.k(b.i64, CCSIDR_L1I), b.k(b.i64, CCSIDR_L2)); + b.put( + a.rt, + b.sel(b.i64, is(0), b.k(b.i64, CCSIDR_L1D), l1_instruction), + ); + } + CntvTvalEl0 | CntpTvalEl0 => { + // The timer value is the signed 32-bit distance to the compare value: + // reads give the low 32 bits of `CVAL - count`, writes `CVAL = count + sext(value)`. + // The physical timer shares the virtual count, as CNTVOFF_EL2 is zero without EL2. + let counter = b.load(b.i64, at + offset_of!(CpuSystem, cntvct_el0)); + let compare = at + + if a.register == CntpTvalEl0 { + offset_of!(CpuSystem, cntp_cval_el0) + } else { + offset_of!(CpuSystem, cntv_cval_el0) + }; + if a.read { + let distance = b.bin(b.i64, B::Sub, b.load(b.i64, compare), counter); + b.put(a.rt, b.imm(b.i64, B::BitAnd, distance, 0xffff_ffff)); + } else { + let delta = b.extend(b.reg(b.i64, a.rt, true), Extend::Sxtw); + b.store(compare, b.bin(b.i64, B::Add, counter, delta)); + } + } + CntpctEl0 => { + // Writes are ignored: the counter is not writable from software. + if a.read { + b.put(a.rt, b.load(b.i64, at + offset_of!(CpuSystem, cntvct_el0))); + } + } + IccIar1El1 => { + // No group 1 interrupt is delivered by this model, so acknowledge reads spurious. + if a.read { + b.put(a.rt, b.k(b.i64, 1023)); + } + } + IccEoir1El1 => {} + Daif => { + let p = at + offset_of!(CpuSystem, daif); + if a.read { + b.put(a.rt, b.imm(b.i64, B::Shl, b.load(b.i64, p), 6)) + } else { + b.store( + p, + b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, b.reg(b.i64, a.rt, true), 6), + 0xf, + ), + ); + } + } + _ => { + if let Some(constant) = super::identification::value(a.register) { + b.put(a.rt, b.k(b.i64, constant)); + return; + } + let offset = + system_offset(a.register).expect("decoded system register has storage"); + let p = at + offset; + if a.read { + let v = if a.register == CntvCtlEl0 { + let counter = b.load(b.i64, at + offset_of!(CpuSystem, cntvct_el0)); + let compare = b.load(b.i64, at + offset_of!(CpuSystem, cntv_cval_el0)); + let reached = b.cmp(C::Uge, counter, compare); + b.bin( + b.i64, + B::BitOr, + b.load(b.i64, p), + b.sel(b.i64, reached, b.k(b.i64, 4), b.k(b.i64, 0)), + ) + } else { + b.load(b.i64, p) + }; + b.put(a.rt, v); + } else { + let from = b.reg(b.i64, a.rt, true); + b.store( + p, + if a.register == CntvCtlEl0 { + b.imm(b.i64, B::BitAnd, from, 3) + } else { + from + }, + ); + } + } + } + } +} +fn system_offset(r: SystemRegister) -> Option { + use SystemRegister::*; + Some(match r { + SpsrEl1 => offset_of!(CpuSystem, spsr_el1), + ElrEl1 => offset_of!(CpuSystem, elr_el1), + EsrEl1 => offset_of!(CpuSystem, esr_el1), + FarEl1 => offset_of!(CpuSystem, far_el1), + ParEl1 => offset_of!(CpuSystem, par_el1), + IsrEl1 => offset_of!(CpuSystem, isr_el1), + IccSreEl1 => offset_of!(CpuSystem, icc_sre_el1), + IccPmrEl1 => offset_of!(CpuSystem, icc_pmr_el1), + IccBpr0El1 => offset_of!(CpuSystem, icc_bpr0_el1), + IccCtlrEl1 => offset_of!(CpuSystem, icc_ctlr_el1), + IccIgrpen0El1 => offset_of!(CpuSystem, icc_igrpen0_el1), + IccIgrpen1El1 => offset_of!(CpuSystem, icc_igrpen1_el1), + IccBpr1El1 => offset_of!(CpuSystem, icc_bpr1_el1), + CntpCtlEl0 => offset_of!(CpuSystem, cntp_ctl_el0), + CntpCvalEl0 => offset_of!(CpuSystem, cntp_cval_el0), + ActlrEl1 => offset_of!(CpuSystem, actlr_el1), + VbarEl1 => offset_of!(CpuSystem, vbar_el1), + SctlrEl2 => offset_of!(CpuSystem, sctlr_el2), + TcrEl2 => offset_of!(CpuSystem, tcr_el2), + Ttbr0El2 => offset_of!(CpuSystem, ttbr0_el2), + Ttbr1El2 => offset_of!(CpuSystem, ttbr1_el2), + MairEl2 => offset_of!(CpuSystem, mair_el2), + VbarEl2 => offset_of!(CpuSystem, vbar_el2), + ElrEl2 => offset_of!(CpuSystem, elr_el2), + SpsrEl2 => offset_of!(CpuSystem, spsr_el2), + EsrEl2 => offset_of!(CpuSystem, esr_el2), + FarEl2 => offset_of!(CpuSystem, far_el2), + HcrEl2 => offset_of!(CpuSystem, hcr_el2), + TpidrEl2 => offset_of!(CpuSystem, tpidr_el2), + CntvoffEl2 => offset_of!(CpuSystem, cntvoff_el2), + CpacrEl1 => offset_of!(CpuSystem, cpacr_el1), + SctlrEl1 => offset_of!(CpuSystem, sctlr_el1), + Ttbr0El1 => offset_of!(CpuSystem, ttbr0_el1), + Ttbr1El1 => offset_of!(CpuSystem, ttbr1_el1), + TcrEl1 => offset_of!(CpuSystem, tcr_el1), + MairEl1 => offset_of!(CpuSystem, mair_el1), + TpidrEl0 => offset_of!(CpuSystem, tpidr_el0), + TpidrroEl0 => offset_of!(CpuSystem, tpidrro_el0), + CntvctEl0 => offset_of!(CpuSystem, cntvct_el0), + CntvCtlEl0 => offset_of!(CpuSystem, cntv_ctl_el0), + CntvCvalEl0 => offset_of!(CpuSystem, cntv_cval_el0), + TpidrEl1 => offset_of!(CpuSystem, tpidr_el1), + ContextidrEl1 => offset_of!(CpuSystem, contextidr_el1), + CsselrEl1 => offset_of!(CpuSystem, csselr_el1), + MdscrEl1 => offset_of!(CpuSystem, mdscr_el1), + CntkctlEl1 => offset_of!(CpuSystem, cntkctl_el1), + OslarEl1 => offset_of!(CpuSystem, oslar_el1), + OsdlrEl1 => offset_of!(CpuSystem, osdlr_el1), + Fpcr => offset_of!(CpuSystem, fpcr), + Fpsr => offset_of!(CpuSystem, fpsr), + Daif => offset_of!(CpuSystem, daif), + Apple(slot) => offset_of!(CpuSystem, apple) + 8 * usize::from(slot), + // After `system` in `Cpu`: relative to it, like every other register here. + AppleBank(slot) => { + offset_of!(Cpu, apple_bank) - offset_of!(Cpu, system) + 8 * usize::from(slot) + } + MidrEl1 => offset_of!(CpuSystem, midr_el1), + IdAa64pfr0El1 => offset_of!(CpuSystem, id_aa64pfr0_el1), + IdAa64mmfr0El1 => offset_of!(CpuSystem, id_aa64mmfr0_el1), + PacKey(slot) => offset_of!(CpuSystem, pac_keys) + 8 * usize::from(slot), + IdAa64isar1El1 => offset_of!(CpuSystem, id_aa64isar1_el1), + _ => return None, + }) +} + +/// EL1 registers that `HCR_EL2.E2H` redirects to their EL2 banks when accessed from EL2: +/// (EL1 field offset, EL2 field offset). +fn vhe_banks(r: SystemRegister) -> Option<(usize, usize)> { + use SystemRegister::*; + Some(match r { + SctlrEl1 => ( + offset_of!(CpuSystem, sctlr_el1), + offset_of!(CpuSystem, sctlr_el2), + ), + Ttbr0El1 => ( + offset_of!(CpuSystem, ttbr0_el1), + offset_of!(CpuSystem, ttbr0_el2), + ), + Ttbr1El1 => ( + offset_of!(CpuSystem, ttbr1_el1), + offset_of!(CpuSystem, ttbr1_el2), + ), + TcrEl1 => ( + offset_of!(CpuSystem, tcr_el1), + offset_of!(CpuSystem, tcr_el2), + ), + MairEl1 => ( + offset_of!(CpuSystem, mair_el1), + offset_of!(CpuSystem, mair_el2), + ), + VbarEl1 => ( + offset_of!(CpuSystem, vbar_el1), + offset_of!(CpuSystem, vbar_el2), + ), + ElrEl1 => ( + offset_of!(CpuSystem, elr_el1), + offset_of!(CpuSystem, elr_el2), + ), + SpsrEl1 => ( + offset_of!(CpuSystem, spsr_el1), + offset_of!(CpuSystem, spsr_el2), + ), + EsrEl1 => ( + offset_of!(CpuSystem, esr_el1), + offset_of!(CpuSystem, esr_el2), + ), + FarEl1 => ( + offset_of!(CpuSystem, far_el1), + offset_of!(CpuSystem, far_el2), + ), + _ => return None, + }) +} +impl Environment for CpuEnvironment { + fn x_offset(&self, r: u8) -> usize { + offset_of!(Cpu, x) + r as usize * 8 + } + fn sp_offset(&self) -> usize { + offset_of!(Cpu, sp) + } + fn flags_offset(&self) -> usize { + offset_of!(Cpu, flags) + } + fn v_offset(&self, r: u8) -> usize { + offset_of!(Cpu, v) + r as usize * 16 + } + fn begin_instruction(&mut self, _: &Builder) { + self.pending += 1; + } + fn exit(&mut self, b: &Builder, next_pc: Value) { + self.leave(b, next_pc) + } + fn memory(&mut self, b: &Builder, pc: u64, access: MemoryAccess<'_>) -> Result<(), LiftError> { + let MemoryAccess { + insn, + address, + writeback, + } = access; + match (*insn, address) { + (Instruction::Memory(a), Some(address)) => self.scalar(b, pc, a, address, writeback), + (Instruction::Literal(a), Some(address)) => self.literal(b, pc, a, address), + (Instruction::Pair(a), Some(address)) => self.pair(b, pc, a, address, writeback), + (Instruction::SimdLiteral(a), Some(address)) => { + self.simd_trap_access(b, address, a.bytes as u64, a.rt, None, None, pc, Trap::Load) + } + (Instruction::SimdPair(a), Some(address)) => { + self.simd_pair(b, pc, a, address, writeback) + } + (Instruction::SimdMemory(a), Some(address)) => { + self.simd_memory(b, pc, a, address, writeback) + } + (Instruction::Exclusive(a), _) => self.exclusive(b, pc, a), + (Instruction::Atomic(a), _) => self.atomic(b, pc, a), + (Instruction::SimdStruct(a), _) => self.simd_struct(b, pc, a), + _ => return Err(LiftError::Unsupported), + } + Ok(()) + } + fn event(&mut self, b: &Builder, pc: u64, insn: &Instruction) -> Result<(), LiftError> { + match *insn { + Instruction::System(a) => self.lower_system(b, a), + Instruction::Clrex(_) => b.byte(offset_of!(Cpu, monitor_valid), 0), + Instruction::Sev | Instruction::Sevl => b.byte(offset_of!(Cpu, event_set), 1), + // Cache maintenance by address has no effect on this machine. + Instruction::CacheOp(_) => {} + Instruction::AddressTranslate(a) => { + b.store(offset_of!(Cpu, address), b.reg(b.i64, a.rt, true)); + b.store( + offset_of!(Cpu, value), + b.k(b.i64, u64::from(a.write) | u64::from(a.user) << 1), + ); + self.trap(b, pc.wrapping_add(4), Trap::AddressTranslate); + } + Instruction::Wfe => self.trap(b, pc.wrapping_add(4), Trap::Wfe), + Instruction::Brk(imm) => { + b.store(offset_of!(Cpu, address), b.k(b.i64, u64::from(imm))); + self.trap(b, pc, Trap::Brk) + } + Instruction::Isb(_) => self.trap(b, pc.wrapping_add(4), Trap::Isb), + Instruction::Tlbi(_) => self.trap(b, pc.wrapping_add(4), Trap::Tlbi), + Instruction::DcZva(rt) => { + b.store(offset_of!(Cpu, address), b.reg(b.i64, rt, true)); + self.trap(b, pc.wrapping_add(4), Trap::DcZva) + } + Instruction::Eret => self.trap(b, pc.wrapping_add(4), Trap::Eret), + Instruction::Svc(imm) => { + b.store(offset_of!(Cpu, address), b.k(b.i64, u64::from(imm))); + self.trap(b, pc.wrapping_add(4), Trap::Svc) + } + Instruction::Psci => self.trap(b, pc.wrapping_add(4), Trap::Psci), + Instruction::Wfi => self.trap(b, pc.wrapping_add(4), Trap::Wfi), + _ => return Err(LiftError::Unsupported), + } + Ok(()) + } +} diff --git a/src/emulator/src/aarch64/exception.rs b/src/emulator/src/aarch64/exception.rs index b0c5260..72657c5 100644 --- a/src/emulator/src/aarch64/exception.rs +++ b/src/emulator/src/aarch64/exception.rs @@ -1,4 +1,4 @@ -use super::cpu::Cpu; +use super::cpu::{Cpu, HCR_TGE}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] @@ -15,6 +15,8 @@ pub struct Reason { pub ec: u8, pub instruction: bool, pub status: u8, + /// Further ISS bits: a BRK's immediate. Syndromes with a DFSC leave this 0. + pub iss: u32, } impl Default for Reason { fn default() -> Self { @@ -23,6 +25,7 @@ impl Default for Reason { ec: 0, instruction: false, status: status::TRANSLATION, + iss: 0, } } } @@ -30,43 +33,62 @@ pub mod status { pub const TRANSLATION: u8 = 0b000100; pub const PERMISSION: u8 = 0b001100; pub const PERMISSION_FETCH: u8 = 0b001101; + /// Synchronous external abort: the access reached no memory or device. + pub const SYNC_EXTERNAL: u8 = 0b010000; } pub fn take(cpu: &mut Cpu, reason: Reason, fault_address: u64) -> u64 { let from = cpu.system.el; - cpu.system.spsr_el1 = (u64::from(cpu.flags) & 0xf000_0000) + let to = if from == 2 || (from == 0 && cpu.system.hcr_el2 & HCR_TGE != 0) { + 2 + } else { + 1 + }; + let spsr = (u64::from(cpu.flags) & 0xf000_0000) | (cpu.system.pan << 22) | ((cpu.system.daif & 15) << 6) | (u64::from(from & 3) << 2) | u64::from(cpu.system.spsel); - cpu.system.elr_el1 = cpu.pc; - cpu.system.esr_el1 = (u64::from(reason.ec & 63) << 26) + let esr = (u64::from(reason.ec & 63) << 26) | (u64::from(reason.instruction) << 25) - | u64::from(reason.status & 63); - cpu.system.far_el1 = fault_address; - let group = if from != 1 { + | u64::from(reason.status & 63) + | u64::from(reason.iss & 0x1ff_ffff); + if to == 2 { + cpu.system.spsr_el2 = spsr; + cpu.system.elr_el2 = cpu.pc; + cpu.system.esr_el2 = esr; + cpu.system.far_el2 = fault_address; + } else { + cpu.system.spsr_el1 = spsr; + cpu.system.elr_el1 = cpu.pc; + cpu.system.esr_el1 = esr; + cpu.system.far_el1 = fault_address; + } + // Same-EL entries pick the vector group by SPSel (SP_EL0 or SP_ELx); entries from a + // lower EL use the lower-EL groups. + let group = if from != to { 2 } else if cpu.system.spsel { 1 } else { 0 }; - let vector = cpu - .system - .vbar_el1 - .wrapping_add(group * 0x200) - .wrapping_add(reason.kind as u64 * 0x80); - let live_stack = if from == 1 && !cpu.system.spsel { - 0 + let vbar = if to == 2 { + cpu.system.vbar_el2 } else { - from & 3 + cpu.system.vbar_el1 }; + let vector = vbar + .wrapping_add(group * 0x200) + .wrapping_add(reason.kind as u64 * 0x80); + // The stack live at the exception is banked: SP_ELx when SPSel was set, else SP_EL0. + let live_stack = if cpu.system.spsel { from & 3 } else { 0 }; cpu.system.sp_el[live_stack as usize] = cpu.sp; - cpu.system.el = 1; + cpu.system.el = to; cpu.system.spsel = true; - cpu.sp = cpu.system.sp_el[1]; + cpu.sp = cpu.system.sp_el[to as usize]; cpu.system.daif = 15; - // SCTLR_EL1.SPAN clear: entry to EL1 sets PAN. - if cpu.system.sctlr_el1 & (1 << 23) == 0 { + // SCTLR_EL1.SPAN clear: entry to EL1 sets PAN. PAN belongs to EL1 only. + if to == 1 && cpu.system.sctlr_el1 & (1 << 23) == 0 { cpu.system.pan = 1; } cpu.monitor_valid = false; @@ -75,23 +97,24 @@ pub fn take(cpu: &mut Cpu, reason: Reason, fault_address: u64) -> u64 { cpu.pc } pub fn eret(cpu: &mut Cpu) -> u64 { - let spsr = cpu.system.spsr_el1; - let from_stack = if cpu.system.el == 1 && !cpu.system.spsel { - 0 + let el = cpu.system.el; + let (spsr, elr) = if el == 2 { + (cpu.system.spsr_el2, cpu.system.elr_el2) } else { - cpu.system.el & 3 + (cpu.system.spsr_el1, cpu.system.elr_el1) }; - cpu.system.sp_el[from_stack as usize] = cpu.sp; + let live_stack = if cpu.system.spsel { el & 3 } else { 0 }; + cpu.system.sp_el[live_stack as usize] = cpu.sp; cpu.monitor_valid = false; let to = ((spsr >> 2) & 3) as u8; cpu.system.el = to; - cpu.system.spsel = to == 1 && spsr & 1 != 0; - let live_stack = if to == 1 && !cpu.system.spsel { 0 } else { to }; - cpu.sp = cpu.system.sp_el[live_stack as usize]; + cpu.system.spsel = to != 0 && spsr & 1 != 0; + let restored = if cpu.system.spsel { to } else { 0 }; + cpu.sp = cpu.system.sp_el[restored as usize]; cpu.system.daif = (spsr >> 6) & 15; cpu.system.pan = (spsr >> 22) & 1; cpu.flags = (spsr & 0xf000_0000) as u32; - cpu.pc = cpu.system.elr_el1; + cpu.pc = elr; cpu.pc } @@ -177,6 +200,77 @@ mod tests { assert_eq!(cpu.system.el, 0); assert_eq!(cpu.sp, 0x2000); } + #[test] + fn exception_from_el2_stays_in_el2_and_banks_only_el2_state() { + let mut cpu = Cpu::default(); + cpu.system.el = 2; + cpu.system.spsel = true; + cpu.sp = 0x7000; + cpu.pc = 0x1234; + cpu.system.vbar_el2 = 0x20000; + cpu.system.vbar_el1 = 0x10000; + cpu.system.elr_el1 = 0xaaaa; + cpu.system.spsr_el1 = 0xbbbb; + // Same EL with SPSel set: the SP_EL2 group, synchronous. + assert_eq!(take(&mut cpu, Reason::default(), 0x55), 0x20200); + assert_eq!(cpu.system.el, 2); + assert_eq!(cpu.system.elr_el2, 0x1234); + assert_eq!(cpu.system.far_el2, 0x55); + assert_eq!((cpu.system.spsr_el2 >> 2) & 3, 2); + assert_eq!(cpu.system.elr_el1, 0xaaaa); + assert_eq!(cpu.system.spsr_el1, 0xbbbb); + assert_eq!(cpu.sp, 0x7000); + } + #[test] + fn eret_from_el2_returns_to_el1h_on_its_stack() { + let mut cpu = Cpu::default(); + cpu.system.el = 2; + cpu.system.spsel = true; + cpu.sp = 0x7000; + cpu.system.sp_el[1] = 0x9000; + cpu.system.spsr_el2 = (1 << 2) | 1; + cpu.system.elr_el2 = 0x4444; + assert_eq!(eret(&mut cpu), 0x4444); + assert_eq!(cpu.system.el, 1); + assert!(cpu.system.spsel); + assert_eq!(cpu.sp, 0x9000); + assert_eq!(cpu.system.sp_el[2], 0x7000); + } + #[test] + fn el0_svc_routes_to_el2_when_hcr_el2_tge_is_set() { + let mut cpu = Cpu::default(); + cpu.system.el = 0; + cpu.system.spsel = false; + cpu.system.hcr_el2 = HCR_TGE; + cpu.pc = 0x1000; + cpu.system.vbar_el2 = 0x20000; + cpu.system.vbar_el1 = 0x10000; + let reason = Reason { + ec: 0x15, + ..Reason::default() + }; + // Lower EL, AArch64, synchronous. + assert_eq!(take(&mut cpu, reason, 0), 0x20400); + assert_eq!(cpu.system.el, 2); + assert_eq!(cpu.system.elr_el2, 0x1000); + assert_eq!(cpu.system.elr_el1, 0); + assert_eq!(cpu.system.esr_el2 >> 26, 0x15); + } + #[test] + fn el0_svc_still_routes_to_el1_without_tge() { + let mut cpu = Cpu::default(); + cpu.system.el = 0; + cpu.system.spsel = false; + cpu.system.vbar_el2 = 0x20000; + cpu.system.vbar_el1 = 0x10000; + let reason = Reason { + ec: 0x15, + ..Reason::default() + }; + assert_eq!(take(&mut cpu, reason, 0), 0x10400); + assert_eq!(cpu.system.el, 1); + assert_eq!(cpu.system.elr_el2, 0); + } } #[cfg(test)] diff --git a/src/emulator/src/aarch64/fp.rs b/src/emulator/src/aarch64/fp.rs index 176b5b6..1f506e1 100644 --- a/src/emulator/src/aarch64/fp.rs +++ b/src/emulator/src/aarch64/fp.rs @@ -1,7 +1,7 @@ //! Scalar floating point (single and double precision) and the vector `fabs`/`fneg` //! forms, executed with host `f32`/`f64` arithmetic. //! -//! Decode only classifies a word with [`parse`]; the JIT stores the word in +//! Decoding lives in `volt_isa_aarch64::fp`; the JIT stores the word in //! `Cpu::value` and traps, and the machine (or user mode) runs [`run`] on the CPU. //! Results follow the Arm rules where they differ from the host's: an invalid //! operation yields the default NaN (positive, quiet), a NaN operand propagates @@ -11,344 +11,8 @@ //! rounding is always to nearest, ties to even), and the `FPSR` exception flags. use super::Cpu; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Precision { - Single, - Double, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Unary { - Move, - Abs, - Neg, - Sqrt, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Binary { - Mul, - Div, - Add, - Sub, - Max, - Min, - MaxNum, - MinNum, - Nmul, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Fused { - Madd, - Msub, - Nmadd, - Nmsub, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Op { - Unary { - kind: Unary, - p: Precision, - rd: u8, - rn: u8, - }, - /// `fcvt`: between single and double; `p` is the source precision. - Convert { - p: Precision, - rd: u8, - rn: u8, - }, - Binary { - kind: Binary, - p: Precision, - rd: u8, - rn: u8, - rm: u8, - }, - Fused { - kind: Fused, - p: Precision, - rd: u8, - rn: u8, - rm: u8, - ra: u8, - }, - /// `fcmp`/`fcmpe`; no `rm` compares with zero. - Compare { - p: Precision, - rn: u8, - rm: Option, - }, - CondCompare { - p: Precision, - rn: u8, - rm: u8, - cond: u32, - nzcv: u32, - }, - Select { - p: Precision, - rd: u8, - rn: u8, - rm: u8, - cond: u32, - }, - Immediate { - p: Precision, - rd: u8, - imm8: u64, - }, - /// `scvtf`/`ucvtf` from a general register. - IntToFloat { - signed: bool, - p: Precision, - wide: bool, - rd: u8, - rn: u8, - }, - /// `fcvtzs`/`fcvtzu` to a general register. - FloatToInt { - signed: bool, - p: Precision, - wide: bool, - rd: u8, - rn: u8, - }, - /// `scvtf`/`ucvtf` between SIMD&FP registers (the source holds an integer). - ScalarIntToFloat { - signed: bool, - p: Precision, - rd: u8, - rn: u8, - }, - /// `fcvtzs`/`fcvtzu` between SIMD&FP registers. - ScalarFloatToInt { - signed: bool, - p: Precision, - rd: u8, - rn: u8, - }, - /// Vector `fabs`/`fneg`. - VectorSign { - negate: bool, - double: bool, - q: bool, - rd: u8, - rn: u8, - }, -} - -fn precision(ty: u32) -> Option { - match ty { - 0 => Some(Precision::Single), - 1 => Some(Precision::Double), - // 3 is half precision, which is not modelled; 2 is reserved. - _ => None, - } -} - -/// The 32- or 64-bit pattern of the 8-bit floating-point immediate `imm8` -/// (`VFPExpandImm`), as `fmov` and the vector `fmov` take it. -pub fn expand_immediate(imm8: u64, double: bool) -> u64 { - let sign = imm8 >> 7; - let b = (imm8 >> 6) & 1; - let fraction = imm8 & 0x3f; - if double { - (sign << 63) | ((b ^ 1) << 62) | (b * 0xff << 54) | (fraction << 48) - } else { - (sign << 31) | ((b ^ 1) << 30) | (b * 0x1f << 25) | (fraction << 19) - } -} - -fn parse(word: u32) -> Option { - let rd = (word & 31) as u8; - let rn = ((word >> 5) & 31) as u8; - let rm = ((word >> 16) & 31) as u8; - let ty = (word >> 22) & 3; - // Scalar floating point: `M=0 S=0 01111 0 type 1 ...`. - if word & 0xff20_0c00 == 0x1e20_0800 { - let kind = match (word >> 12) & 15 { - 0 => Binary::Mul, - 1 => Binary::Div, - 2 => Binary::Add, - 3 => Binary::Sub, - 4 => Binary::Max, - 5 => Binary::Min, - 6 => Binary::MaxNum, - 7 => Binary::MinNum, - 8 => Binary::Nmul, - _ => return None, - }; - return Some(Op::Binary { - kind, - p: precision(ty)?, - rd, - rn, - rm, - }); - } - // One-source: bits 14:10 = 10000; the opcode (bits 20:15) includes bit 15. - if word & 0xff20_7c00 == 0x1e20_4000 { - let p = precision(ty)?; - let kind = match (word >> 15) & 0x3f { - 0 => Unary::Move, - 1 => Unary::Abs, - 2 => Unary::Neg, - 3 => Unary::Sqrt, - // `fcvt` to single from double, to double from single. - 4 if p == Precision::Double => return Some(Op::Convert { p, rd, rn }), - 5 if p == Precision::Single => return Some(Op::Convert { p, rd, rn }), - _ => return None, - }; - return Some(Op::Unary { kind, p, rd, rn }); - } - if word & 0xff20_fc07 == 0x1e20_2000 { - let zero = word & 8 != 0; - return Some(Op::Compare { - p: precision(ty)?, - rn, - rm: (!zero).then_some(rm), - }); - } - if word & 0xff20_0c10 == 0x1e20_0400 { - return Some(Op::CondCompare { - p: precision(ty)?, - rn, - rm, - cond: (word >> 12) & 15, - nzcv: word & 15, - }); - } - if word & 0xff20_0c00 == 0x1e20_0c00 { - return Some(Op::Select { - p: precision(ty)?, - rd, - rn, - rm, - cond: (word >> 12) & 15, - }); - } - if word & 0xff20_1fe0 == 0x1e20_1000 { - return Some(Op::Immediate { - p: precision(ty)?, - rd, - imm8: u64::from((word >> 13) & 0xff), - }); - } - if word & 0xff00_0000 == 0x1f00_0000 { - let kind = match ((word >> 21) & 1, (word >> 15) & 1) { - (0, 0) => Fused::Madd, - (0, _) => Fused::Msub, - (_, 0) => Fused::Nmadd, - _ => Fused::Nmsub, - }; - return Some(Op::Fused { - kind, - p: precision(ty)?, - rd, - rn, - rm, - ra: ((word >> 10) & 31) as u8, - }); - } - // Conversions between floating point and general registers. The `fmov` forms - // (rmode 0, opcode 6 and 7) are `SimdFmov`. - if word & 0x7f20_fc00 == 0x1e20_0000 { - let (p, wide) = (precision(ty)?, word & 0x8000_0000 != 0); - return match ((word >> 19) & 3, (word >> 16) & 7) { - (0, 2) => Some(Op::IntToFloat { - signed: true, - p, - wide, - rd, - rn, - }), - (0, 3) => Some(Op::IntToFloat { - signed: false, - p, - wide, - rd, - rn, - }), - (3, 0) => Some(Op::FloatToInt { - signed: true, - p, - wide, - rd, - rn, - }), - (3, 1) => Some(Op::FloatToInt { - signed: false, - p, - wide, - rd, - rn, - }), - _ => None, - }; - } - // The same conversions between SIMD&FP registers: bit 22 is `sz`. - let p = if word & 0x0040_0000 != 0 { - Precision::Double - } else { - Precision::Single - }; - match word & 0xffbf_fc00 { - 0x5e21_d800 => { - return Some(Op::ScalarIntToFloat { - signed: true, - p, - rd, - rn, - }); - } - 0x7e21_d800 => { - return Some(Op::ScalarIntToFloat { - signed: false, - p, - rd, - rn, - }); - } - 0x5ea1_b800 => { - return Some(Op::ScalarFloatToInt { - signed: true, - p, - rd, - rn, - }); - } - 0x7ea1_b800 => { - return Some(Op::ScalarFloatToInt { - signed: false, - p, - rd, - rn, - }); - } - _ => {} - } - // Vector `fabs` (U=0) and `fneg` (U=1). - if word & 0x9fbf_fc00 == 0x0ea0_f800 { - let (q, double) = (word & 0x4000_0000 != 0, word & 0x0040_0000 != 0); - // `.1d` is reserved. - if double && !q { - return None; - } - return Some(Op::VectorSign { - negate: word & 0x2000_0000 != 0, - double, - q, - rd, - rn, - }); - } - None -} - -/// Whether [`run`] implements `word`. -pub fn is_supported(word: u32) -> bool { - parse(word).is_some() -} +use volt_isa_aarch64::decode::expand_immediate; +use volt_isa_aarch64::fp::{Binary, Fused, Op, Precision, Unary}; trait Float: Copy + PartialOrd { const SIGN: u64; @@ -550,10 +214,8 @@ macro_rules! at { }; } -/// Run the floating-point instruction `word` on `cpu`. `word` must satisfy -/// [`is_supported`]. -pub fn run(cpu: &mut Cpu, word: u32) { - let op = parse(word).expect("an unsupported floating-point word reached the executor"); +/// Run the decoded floating-point instruction `op` on `cpu`. +pub fn run(cpu: &mut Cpu, op: Op) { match op { Op::Unary { kind, p, rd, rn } => { let a = read(cpu, p, rn); @@ -614,7 +276,8 @@ pub fn run(cpu: &mut Cpu, word: u32) { at!(p, T => fused(kind, T::from_u64(n), T::from_u64(m), T::from_u64(a)).to_u64()); write(cpu, rd, r); } - Op::Compare { p, rn, rm } => { + // FPSR exception flags are not modelled, so `fcmpe` executes as `fcmp`. + Op::Compare { p, rn, rm, .. } => { let a = read(cpu, p, rn); let b = rm.map_or(0, |rm| read(cpu, p, rm)); let nzcv = at!(p, T => compare(T::from_u64(a), T::from_u64(b))); @@ -740,6 +403,11 @@ fn float_to_int(bits: u64, signed: bool, wide: bool, p: Precision) -> u64 { mod tests { use super::*; + /// Decode `word` and run it, as the machine does for a trapped host word. + fn run(cpu: &mut Cpu, word: u32) { + super::run(cpu, volt_isa_aarch64::fp::decode(word).expect("an fp word")); + } + fn cpu() -> Cpu { Cpu::default() } @@ -978,19 +646,6 @@ mod tests { assert_eq!(c.v[0], [0x3f80_0000_4000_0000, 0]); } - #[test] - fn unsupported_encodings_are_not_claimed() { - // fmov d0, x1 and fmov x0, d1 are `SimdFmov`, not conversions. - assert!(!is_supported(0x9e670020)); - assert!(!is_supported(0x9e660020)); - // Half precision (type 3) and reserved type 2. - assert!(!is_supported(0x1ee22820)); - assert!(!is_supported(0x1ea22820)); - // fneg v0.1d is reserved. - assert!(!is_supported(0x2ee0f820)); - assert!(is_supported(0x1e620820)); - } - #[test] fn the_immediate_expansion_matches_the_architectural_pattern() { assert_eq!(expand_immediate(0x70, false), 0x3f80_0000); diff --git a/src/emulator/src/aarch64/gxf.rs b/src/emulator/src/aarch64/gxf.rs new file mode 100644 index 0000000..9fd9f9f --- /dev/null +++ b/src/emulator/src/aarch64/gxf.rs @@ -0,0 +1,55 @@ +//! Guarded execution (GXF): `genter` and `gexit`, the instructions that move between +//! EL and the guarded levels GL1 and GL2 (Asahi Linux documentation, "Apple Proprietary +//! Instructions" and "SPRR and GXF"). Decoding lives in `volt_isa_aarch64::gxf`. +//! +//! The feature is off by default and only a monitor firmware (SPTM or TXM) enables it, +//! which this model does not run. Its enable bit reads as zero, so both instructions +//! take an undefined-instruction exception to EL1. The documentation says they "fault by +//! default" without naming the exception class; undefined-instruction is an assumption. + +use super::{Cpu, exception}; +use volt_isa_aarch64::gxf::Op; + +/// Take the undefined-instruction exception `genter` or `gexit` raises while GXF is off. +/// `cpu.pc` is the address of the next instruction; the exception returns to the +/// instruction itself. +pub fn run(cpu: &mut Cpu, _op: Op) { + cpu.pc = cpu.pc.wrapping_sub(4); + let pc = cpu.pc; + exception::take( + cpu, + exception::Reason { + ec: 0, + instruction: true, + status: 0, + ..Default::default() + }, + pc, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn with_guarded_execution_off_genter_is_an_undefined_instruction_at_the_instruction() { + let mut cpu = Cpu::default(); + cpu.system.el = 1; + cpu.system.spsel = true; + cpu.system.vbar_el1 = 0x4000_1000; + // The executor runs after the instruction at 0x4000_0000 has advanced `pc`. + cpu.pc = 0x4000_0004; + run(&mut cpu, Op::Genter { imm5: 0 }); + assert_eq!( + cpu.system.elr_el1, 0x4000_0000, + "returns to the genter itself" + ); + assert_eq!(cpu.system.esr_el1 >> 26, 0, "EC is unknown reason"); + assert_eq!( + cpu.pc, + 0x4000_1000 + 0x200, + "current EL with SP_ELx, synchronous" + ); + } +} diff --git a/src/emulator/src/aarch64/host.rs b/src/emulator/src/aarch64/host.rs index e884992..bfcf0fb 100644 --- a/src/emulator/src/aarch64/host.rs +++ b/src/emulator/src/aarch64/host.rs @@ -1,19 +1,110 @@ //! Instructions the JIT does not compile. It stores the raw word in `Cpu::value` //! and traps; the machine (or user mode) then runs the word on the host here. //! Floating point and the wider integer SIMD families live in [`fp`] and [`simd`]. +//! +//! Which words these are is decided by the one decoder, `volt_isa_aarch64::decode`: +//! the instruction variants [`runs_on_host`] names. -use super::{Cpu, fp, simd}; +use super::{Cpu, fp, gxf, pauth, simd}; +use volt_isa_aarch64::decode::{Instruction, decode}; -/// Whether [`run`] implements `word`. -pub fn is_supported(word: u32) -> bool { - fp::is_supported(word) || simd::is_supported(word) +/// Whether the emulator runs `instruction` on the host instead of lifting it. +pub fn runs_on_host(instruction: &Instruction) -> bool { + matches!( + instruction, + Instruction::Fp(_) | Instruction::Simd(_) | Instruction::Pauth(_) | Instruction::Gxf(_) + ) } -/// Run `word` on `cpu`. `word` must satisfy [`is_supported`]. +/// Whether the emulator ends a translated block after `instruction`. +/// +/// This is policy of this emulator's [`CpuEnvironment`](super::environment), not a +/// property of the instruction set: everything the environment forwards to the +/// machine (memory, exclusives, atomics, system events, words run on the host) must +/// leave the block so the machine can run it, and so must every architectural +/// branch. With `inline_memory`, plain loads and stores are compiled with an inline +/// data-TLB fast path whose slow path exits mid-block, so they no longer end it. +pub fn ends_block(instruction: &Instruction, inline_memory: bool) -> bool { + use Instruction::*; + if runs_on_host(instruction) { + return true; + } + if inline_memory + && matches!( + instruction, + Memory(_) | Pair(_) | Literal(_) | SimdMemory(_) | SimdPair(_) + ) + { + return false; + } + matches!( + instruction, + Memory(_) + | Exclusive(_) + | Atomic(_) + | Literal(_) + | SimdLiteral(_) + | Pair(_) + | SimdPair(_) + | SimdMemory(_) + | SimdStruct(_) + | B(_) + | BCond(_) + | TestBranch(_) + | Call(_) + | Indirect(_) + | Svc(_) + | Psci + | Wfi + | Wfe + | Brk(_) + | Isb(_) + | DcZva(_) + | AddressTranslate(_) + | Eret + | Tlbi(_) + ) +} + +/// Run `word` on `cpu`. `word` must decode to an instruction [`runs_on_host`] names. pub fn run(cpu: &mut Cpu, word: u32) { - if fp::is_supported(word) { - fp::run(cpu, word) - } else { - simd::run(cpu, word) + match decode(word) { + Ok(Instruction::Fp(op)) => fp::run(cpu, op), + Ok(Instruction::Simd(op)) => simd::run(cpu, op), + Ok(Instruction::Pauth(op)) => pauth::run(cpu, op), + Ok(Instruction::Gxf(op)) => gxf::run(cpu, op), + other => unreachable!("word {word:08x} is not a host instruction: {other:?}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_policy_cuts_at_machine_effects_and_branches() { + let word = |w| decode(w).unwrap(); + assert!(!ends_block(&word(0xd503309f), false)); // dsb + assert!(ends_block(&word(0xd5033fdf), false)); // isb + assert!(ends_block(&word(0xd69f03e0), false)); // eret + assert!(ends_block(&word(0x1400_0000), false)); // b . + assert!(ends_block(&word(0x1e22_2820), true)); // fadd s0, s1, s2: runs on the host + } + + #[test] + fn inline_memory_only_lets_plain_accesses_continue() { + let load = decode(0xf940_0020).unwrap(); // ldr x0, [x1] + let exclusive = decode(0xc85f_7c20).unwrap(); // ldxr x0, [x1] + assert!(ends_block(&load, false)); + assert!(!ends_block(&load, true)); + assert!(ends_block(&exclusive, true)); + } + + #[test] + fn floating_point_pointer_authentication_and_integer_simd_run_on_the_host() { + assert!(runs_on_host(&decode(0x1e22_2820).unwrap())); // fadd s0, s1, s2 + assert!(runs_on_host(&decode(0xdac1_0020).unwrap())); // pacia x0, x1 + assert!(runs_on_host(&decode(0x4e82_1820).unwrap())); // uzp1 v0.4s, v1.4s, v2.4s + assert!(!runs_on_host(&decode(0x8b02_0020).unwrap())); // add x0, x1, x2: lifted } } diff --git a/src/emulator/src/aarch64/identification.rs b/src/emulator/src/aarch64/identification.rs index 46a7ba2..67a47d0 100644 --- a/src/emulator/src/aarch64/identification.rs +++ b/src/emulator/src/aarch64/identification.rs @@ -1,4 +1,4 @@ -use super::decode::SystemRegister; +use volt_isa_aarch64::decode::SystemRegister; #[allow(non_upper_case_globals)] pub const counter_hz: u64 = 62_500_000; /// Cache hierarchy `CLIDR_EL1` reports: split L1, unified L2, 64-byte lines. @@ -16,21 +16,23 @@ pub const CCSIDR_L1D: u64 = ccsidr(4, 128, 4); pub const CCSIDR_L1I: u64 = ccsidr(2, 128, 4); /// L2 unified: 1 MiB, 8-way. pub const CCSIDR_L2: u64 = ccsidr(4, 2048, 8); +/// The virt board's `MIDR_EL1`, `ID_AA64PFR0_EL1` and `ID_AA64MMFR0_EL1`. A board +/// may advertise different identity, so `value` leaves these three to the CPU state. +pub const DEFAULT_MIDR_EL1: u64 = 0x410f_d0f0; +pub const DEFAULT_ID_AA64PFR0_EL1: u64 = (0xf << 20) | (0xf << 16) | (1 << 4) | 1; +pub const DEFAULT_ID_AA64MMFR0_EL1: u64 = (0xf << 24) | 2; +pub const DEFAULT_ID_AA64ISAR1_EL1: u64 = 1 << 20; pub fn value(register: SystemRegister) -> Option { use SystemRegister::*; Some(match register { - MidrEl1 => 0x410f_d0f0, MpidrEl1 => 0x8000_0000, CtrEl0 => { (1 << 31) | (1 << 29) | (1 << 28) | (4 << 24) | (4 << 20) | (4 << 16) | (3 << 14) | 4 } DczidEl0 => 4, CntfrqEl0 => counter_hz, - IdAa64pfr0El1 => (0xf << 20) | (0xf << 16) | (1 << 4) | 1, IdAa64dfr0El1 => 6, - IdAa64mmfr0El1 => (0xf << 24) | 2, IdAa64isar0El1 => 2 << 20, - IdAa64isar1El1 => 1 << 20, ClidrEl1 => CLIDR, RevidrEl1 | IdAa64pfr1El1 | IdAa64pfr2El1 | IdAa64zfr0El1 | IdAa64smfr0El1 | IdAa64fpfr0El1 | IdAa64isar3El1 | AidrEl1 | IdAa64dfr1El1 | IdAa64isar2El1 @@ -47,17 +49,17 @@ mod tests { use SystemRegister::*; assert_eq!(value(CntfrqEl0), Some(62_500_000)); assert_eq!(value(DczidEl0), Some(4)); - let pfr0 = value(IdAa64pfr0El1).unwrap(); + let pfr0 = DEFAULT_ID_AA64PFR0_EL1; assert_eq!(pfr0 & 0xffff, 0x11); assert_eq!((pfr0 >> 16) & 0xff, 0xff); assert_eq!((pfr0 >> 24) & 15, 0); - let mmfr0 = value(IdAa64mmfr0El1).unwrap(); + let mmfr0 = DEFAULT_ID_AA64MMFR0_EL1; assert_eq!(mmfr0 & 15, 2); assert_eq!((mmfr0 >> 24) & 15, 15); assert_eq!((mmfr0 >> 20) & 15, 0); // FEAT_LSE (Atomics = 2) and FEAT_LRCPC (LDAPR) are implemented. assert_eq!(value(IdAa64isar0El1), Some(2 << 20)); - assert_eq!(value(IdAa64isar1El1), Some(1 << 20)); + assert_eq!(DEFAULT_ID_AA64ISAR1_EL1, 1 << 20); assert_eq!(value(MpidrEl1), Some(1 << 31)); assert_eq!(value(Ttbr0El1), None); } diff --git a/src/emulator/src/aarch64/mod.rs b/src/emulator/src/aarch64/mod.rs index db05489..053b6fd 100644 --- a/src/emulator/src/aarch64/mod.rs +++ b/src/emulator/src/aarch64/mod.rs @@ -2,15 +2,16 @@ pub mod ahead; pub mod cache; pub mod compile; pub mod cpu; -pub mod decode; +mod environment; pub mod exception; pub mod fp; +pub mod gxf; pub mod host; pub mod identification; +pub mod pauth; pub mod simd; pub mod simd_struct; pub mod translate; pub use cache::Cache; pub use cpu::Cpu; -pub use decode::Instruction; diff --git a/src/emulator/src/aarch64/pauth.rs b/src/emulator/src/aarch64/pauth.rs new file mode 100644 index 0000000..3fed138 --- /dev/null +++ b/src/emulator/src/aarch64/pauth.rs @@ -0,0 +1,539 @@ +//! Pointer authentication (FEAT_PAuth): the QARMA5 `ComputePAC` the architecture +//! defines, the sign, authenticate and strip operations on 48-bit pointers, and the +//! PAuth instructions the JIT hands to the host ([`super::host`]); decoding lives in +//! `volt_isa_aarch64::pauth`. +//! +//! The emulated CPU advertises the architected QARMA5 algorithm without TBI, EPAC or +//! FPAC (`ID_AA64ISAR1_EL1.APA` = 1). A pointer's PAC then occupies bits 63:56 and 54:48, +//! and authentication failure corrupts bits 62:61 instead of faulting. +//! +//! The cipher follows the pseudocode in the architecture as ported from QEMU's +//! `target/arm/tcg/pauth_helper.c`; the tests check it against that implementation. + +use super::{Cpu, exception}; +use volt_isa_aarch64::pauth::{KeyId, Modifier, Op}; + +/// A 128-bit PAuth key: `hi` is bits 127:64 (`APxAKeyHi_EL1`), `lo` bits 63:0. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Key { + pub hi: u64, + pub lo: u64, +} + +/// The 48-bit pointer's bits 63:48 all equal bit 47, so the pointer is canonical. +fn is_canonical(ptr: u64) -> bool { + let top = ptr >> 48; + top == 0 || top == 0xffff +} + +/// Bit 55 selects the address range. A canonical pointer's bits 63:48 equal it. +fn original_ptr(ptr: u64) -> u64 { + if ptr & (1 << 55) != 0 { + ptr | 0xffff_0000_0000_0000 + } else { + ptr & 0x0000_ffff_ffff_ffff + } +} + +/// The PAC of a pointer: the `ComputePAC` of its extension bits filled in. +pub fn add_pac(ptr: u64, modifier: u64, key: Key) -> u64 { + let ext = (ptr >> 63) & 1; + let ext_ptr = (ptr & 0x0000_ffff_ffff_ffff) + | if (ptr >> 63) & 1 != 0 { + 0xffff_0000_0000_0000 + } else { + 0 + }; + let mut pac = compute_pac(ext_ptr, modifier, key); + if !is_canonical(ptr) { + pac ^= 1 << 62; + } + let pac = pac & !((1 << 55) | 0x0000_ffff_ffff_ffff); + pac | (ext << 55) | (ptr & 0x0000_ffff_ffff_ffff) +} + +/// Authenticate `ptr` against `modifier`. On success the pointer is returned with its +/// PAC removed. On failure its bits 62:61 carry an error code and it is non-canonical, +/// so its next use faults. +pub fn auth(ptr: u64, modifier: u64, key: Key, key_number: u64) -> u64 { + let orig = original_ptr(ptr); + let pac = compute_pac(orig, modifier, key); + let cmp_mask = 0xff7f_0000_0000_0000u64; + if (pac ^ ptr) & cmp_mask != 0 { + let error = (key_number << 1) | (key_number ^ 1); + return (orig & !(3 << 61)) | ((error & 3) << 61); + } + orig +} + +/// Remove the PAC from a pointer. +pub fn strip(ptr: u64) -> u64 { + original_ptr(ptr) +} + +/// `PACGA`: a 32-bit generic MAC in the top half of the result. +pub fn pacga(data: u64, modifier: u64, key: Key) -> u64 { + compute_pac(data, modifier, key) & 0xffff_ffff_0000_0000 +} + +const RC: [u64; 5] = [ + 0x0000_0000_0000_0000, + 0x1319_8A2E_0370_7344, + 0xA409_3822_299F_31D0, + 0x082E_FA98_EC4E_6C89, + 0x4528_21E6_38D0_1377, +]; +const ALPHA: u64 = 0xC0AC_29B7_C97C_50DD; +const SBOX: [u8; 16] = [ + 0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe, 0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa, +]; +const INV_SBOX: [u8; 16] = [ + 0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9, 0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3, +]; + +/// Cell permutations as `(source nibble, destination nibble)`. +const CELL_SHUFFLE: [(u8, u8); 16] = [ + (13, 0), + (6, 1), + (11, 2), + (0, 3), + (7, 4), + (12, 5), + (1, 6), + (10, 7), + (8, 8), + (3, 9), + (14, 10), + (5, 11), + (2, 12), + (9, 13), + (4, 14), + (15, 15), +]; +const CELL_INV_SHUFFLE: [(u8, u8); 16] = [ + (3, 0), + (6, 1), + (12, 2), + (9, 3), + (14, 4), + (11, 5), + (1, 6), + (4, 7), + (8, 8), + (13, 9), + (7, 10), + (2, 11), + (5, 12), + (0, 13), + (10, 14), + (15, 15), +]; +/// The tweak permutations, with the cells the tweak LFSR steps marked. +const TWEAK_SHUFFLE: [(u8, u8, bool); 16] = [ + (4, 0, false), + (5, 1, false), + (6, 2, true), + (7, 3, false), + (11, 4, true), + (2, 5, false), + (3, 6, false), + (8, 7, true), + (12, 8, false), + (13, 9, false), + (14, 10, false), + (15, 11, true), + (0, 12, true), + (1, 13, false), + (10, 14, true), + (9, 15, true), +]; +const TWEAK_INV_SHUFFLE: [(u8, u8, bool); 16] = [ + (12, 0, true), + (13, 1, false), + (5, 2, false), + (6, 3, false), + (0, 4, false), + (1, 5, false), + (2, 6, true), + (3, 7, false), + (7, 8, true), + (15, 9, true), + (14, 10, true), + (4, 11, true), + (8, 12, false), + (9, 13, false), + (10, 14, false), + (11, 15, true), +]; + +fn nibble(value: u64, n: u8) -> u64 { + (value >> (4 * n)) & 0xf +} + +fn permute(value: u64, table: &[(u8, u8)]) -> u64 { + table + .iter() + .fold(0, |out, &(src, dst)| out | nibble(value, src) << (4 * dst)) +} + +fn permute_tweak(value: u64, table: &[(u8, u8, bool)], step: fn(u64) -> u64) -> u64 { + table.iter().fold(0, |out, &(src, dst, stepped)| { + let cell = nibble(value, src); + let cell = if stepped { step(cell) } else { cell }; + out | cell << (4 * dst) + }) +} + +fn cell_shuffle(value: u64) -> u64 { + permute(value, &CELL_SHUFFLE) +} + +fn cell_inv_shuffle(value: u64) -> u64 { + permute(value, &CELL_INV_SHUFFLE) +} + +fn tweak_cell_rot(cell: u64) -> u64 { + (cell >> 1) | (((cell ^ (cell >> 1)) & 1) << 3) +} + +fn tweak_cell_inv_rot(cell: u64) -> u64 { + ((cell << 1) & 0xf) | ((cell & 1) ^ (cell >> 3)) +} + +fn tweak_shuffle(value: u64) -> u64 { + permute_tweak(value, &TWEAK_SHUFFLE, tweak_cell_rot) +} + +fn tweak_inv_shuffle(value: u64) -> u64 { + permute_tweak(value, &TWEAK_INV_SHUFFLE, tweak_cell_inv_rot) +} + +fn rot_cell(cell: u64, n: u32) -> u64 { + let doubled = cell | cell << 4; + (doubled >> (4 - n)) & 0xf +} + +/// The MixColumns-like diffusion layer, applied to each column of the 4x4 cell matrix. +fn mult(value: u64) -> u64 { + let mut out = 0; + for column in 0..4u8 { + let i0 = nibble(value, column); + let i4 = nibble(value, column + 4); + let i8 = nibble(value, column + 8); + let ic = nibble(value, column + 12); + let t0 = rot_cell(i8, 1) ^ rot_cell(i4, 2) ^ rot_cell(i0, 1); + let t1 = rot_cell(ic, 1) ^ rot_cell(i4, 1) ^ rot_cell(i0, 2); + let t2 = rot_cell(ic, 2) ^ rot_cell(i8, 1) ^ rot_cell(i0, 1); + let t3 = rot_cell(ic, 1) ^ rot_cell(i8, 2) ^ rot_cell(i4, 1); + out |= t3 << (4 * column); + out |= t2 << (4 * (column + 4)); + out |= t1 << (4 * (column + 8)); + out |= t0 << (4 * (column + 12)); + } + out +} + +fn sub(value: u64, table: &[u8; 16]) -> u64 { + (0..16).fold(0, |out, n| { + out | u64::from(table[nibble(value, n) as usize]) << (4 * n) + }) +} + +/// `ComputePAC` of the architecture: the QARMA5 cipher over `data`, keyed by `key`, +/// tweaked by `modifier`. +pub fn compute_pac(data: u64, modifier: u64, key: Key) -> u64 { + let key0 = key.hi; + let key1 = key.lo; + let modk0 = (key0 << 63) | ((key0 >> 1) ^ (key0 >> 63)); + let mut runningmod = modifier; + let mut working = data ^ key0; + for (i, rc) in RC.iter().enumerate() { + working ^= key1 ^ runningmod; + working ^= rc; + if i > 0 { + working = mult(cell_shuffle(working)); + } + working = sub(working, &SBOX); + runningmod = tweak_shuffle(runningmod); + } + working ^= modk0 ^ runningmod; + working = mult(cell_shuffle(working)); + working = sub(working, &SBOX); + working = mult(cell_shuffle(working)); + working ^= key1; + working = sub(cell_inv_shuffle(working), &INV_SBOX); + working = cell_inv_shuffle(mult(working)); + working ^= key0 ^ runningmod; + for i in 0..=4usize { + working = sub(working, &INV_SBOX); + if i < 4 { + working = cell_inv_shuffle(mult(working)); + } + runningmod = tweak_inv_shuffle(runningmod); + working ^= RC[4 - i] ^ (key1 ^ runningmod) ^ ALPHA; + } + working ^ modk0 +} + +/// Index of `id` into `System::pac_keys` (two words per key, low word first). +fn key_slot(id: KeyId) -> usize { + match id { + KeyId::Ia => 0, + KeyId::Ib => 1, + KeyId::Da => 2, + KeyId::Db => 3, + KeyId::Ga => 4, + } +} + +fn key(cpu: &Cpu, id: KeyId) -> Key { + let keys = &cpu.system.pac_keys; + Key { + lo: keys[2 * key_slot(id)], + hi: keys[2 * key_slot(id) + 1], + } +} + +/// Whether the key is enabled for the current translation regime (`SCTLR_EL1`). +fn enabled(cpu: &Cpu, id: KeyId) -> bool { + id.enable_bit() + .is_none_or(|bit| cpu.system.sctlr_el1 & (1 << bit) != 0) +} + +/// Register `n` read as `Xn|SP` (31 is the stack pointer). +fn read_sp(cpu: &Cpu, n: u8) -> u64 { + if n == 31 { cpu.sp } else { cpu.x[n as usize] } +} + +/// Register `n` read as `Xn` (31 is zero). +fn read_zr(cpu: &Cpu, n: u8) -> u64 { + if n == 31 { 0 } else { cpu.x[n as usize] } +} + +fn write_x(cpu: &mut Cpu, n: u8, value: u64) { + if n != 31 { + cpu.x[n as usize] = value; + } +} + +fn modifier_value(cpu: &Cpu, modifier: Modifier) -> u64 { + match modifier { + Modifier::Zero => 0, + Modifier::Reg(n) => read_sp(cpu, n), + } +} + +/// Execute the decoded PAuth instruction `op` on `cpu`. +/// `cpu.pc` is the address of the next instruction, which a link writes to `LR`. +pub fn run(cpu: &mut Cpu, op: Op) { + match op { + // `hint` only selects the disassembly spelling (`paciasp` vs `pacia x30, sp`). + Op::Sign { + key: id, + reg, + modifier, + hint: _, + } => { + if enabled(cpu, id) { + let value = add_pac( + cpu.x[reg as usize], + modifier_value(cpu, modifier), + key(cpu, id), + ); + write_x(cpu, reg, value); + } + } + Op::Auth { + key: id, + reg, + modifier, + hint: _, + } => { + if enabled(cpu, id) { + let value = auth( + cpu.x[reg as usize], + modifier_value(cpu, modifier), + key(cpu, id), + id.number(), + ); + write_x(cpu, reg, value); + } + } + // `xpaci` and `xpacd` strip identically here. + Op::Strip { reg, .. } => { + let value = strip(cpu.x[reg as usize]); + write_x(cpu, reg, value); + } + Op::Pacga { rd, rn, modifier } => { + let value = pacga( + read_zr(cpu, rn), + modifier_value(cpu, modifier), + key(cpu, KeyId::Ga), + ); + write_x(cpu, rd, value); + } + Op::Branch { + key: id, + link, + target, + modifier, + } => { + let next = cpu.pc; + let mut destination = read_zr(cpu, target); + if enabled(cpu, id) { + destination = auth( + destination, + modifier_value(cpu, modifier), + key(cpu, id), + id.number(), + ); + } + if link { + write_x(cpu, 30, next); + } + cpu.pc = destination; + } + Op::Return { key: id } => { + let mut destination = cpu.x[30]; + if enabled(cpu, id) { + destination = auth(destination, cpu.sp, key(cpu, id), id.number()); + } + cpu.pc = destination; + } + Op::ExceptionReturn { key: id } => { + let mut elr = cpu.system.elr_el1; + if enabled(cpu, id) { + elr = auth(elr, cpu.sp, key(cpu, id), id.number()); + } + cpu.system.elr_el1 = elr; + cpu.pc = exception::eret(cpu); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The key the QEMU reference was run with. + const KEY: Key = Key { + hi: 0xec28_02d4_e0a4_88e9, + lo: 0x84be_85ce_9804_e94b, + }; + + fn op(word: u32) -> Op { + volt_isa_aarch64::pauth::decode(word).unwrap() + } + + /// `ComputePAC` values from QEMU's `pauth_computepac_architected`, compiled from its + /// source with the same key: `(data, modifier, pac)`. + const REFERENCE: [(u64, u64, u64); 6] = [ + ( + 0x0000_0000_0000_0000, + 0x0000_0000_0000_0000, + 0xbc2f_6e1c_6e1a_f447, + ), + ( + 0xffff_fe00_0700_4000, + 0x477d_469d_ec0b_8762, + 0xc1b6_9701_e911_0f0c, + ), + ( + 0x0000_0008_1234_5678, + 0x0000_0000_0000_0000, + 0xc3ad_cffe_19ff_629c, + ), + ( + 0x0000_0008_1234_5678, + 0x2222_2222_2222_2222, + 0xfe68_3d52_004b_603d, + ), + ( + 0x1234_5678_1234_5678, + 0x0000_0000_0000_0000, + 0x29e3_4d27_b244_88a6, + ), + ( + 0x1234_5678_1234_5678, + 0x477d_469d_ec0b_8762, + 0x2b7a_744b_38ed_59f6, + ), + ]; + + #[test] + fn compute_pac_matches_the_architected_reference() { + for (data, modifier, pac) in REFERENCE { + assert_eq!(compute_pac(data, modifier, KEY), pac, "data {data:#x}"); + } + } + + #[test] + fn sign_then_authenticate_returns_the_pointer_and_strip_removes_the_pac() { + let ptr = 0x0000_0008_1234_5678; + let signed = add_pac(ptr, 0x2222, KEY); + assert_ne!(signed, ptr, "a PAC is inserted"); + assert_eq!(auth(signed, 0x2222, KEY, 0), ptr); + assert_eq!(strip(signed), ptr); + } + + #[test] + fn a_kernel_pointer_round_trips_through_sign_and_auth() { + let ptr = 0xffff_8000_8004_0000; + let signed = add_pac(ptr, 0x1234, KEY); + assert_eq!(auth(signed, 0x1234, KEY, 2), ptr); + } + + #[test] + fn a_wrong_modifier_or_key_leaves_a_non_canonical_pointer() { + let ptr = 0x0000_0008_1234_5678; + let signed = add_pac(ptr, 0x2222, KEY); + let other_key = Key { + lo: KEY.lo ^ 1, + ..KEY + }; + for failed in [ + auth(signed, 0x3333, KEY, 0), + auth(signed, 0x2222, other_key, 0), + ] { + assert!(!is_canonical(failed), "{failed:#x} must not be usable"); + assert_eq!((failed >> 61) & 3, 1, "error code for key A"); + } + } + + #[test] + fn a_non_canonical_input_is_not_signed_into_a_valid_pointer() { + // A pointer whose extension bits are wrong gets a corrupted PAC, so it cannot + // authenticate later. + let bad = 0x1234_0008_1234_5678; + let signed = add_pac(bad, 0, KEY); + assert!(!is_canonical(auth(signed, 0, KEY, 0))); + } + + #[test] + fn pacga_keeps_only_the_top_word() { + let mac = pacga(0x1234, 0x5678, KEY); + assert_eq!(mac & 0xffff_ffff, 0); + assert_eq!( + mac, + compute_pac(0x1234, 0x5678, KEY) & 0xffff_ffff_0000_0000 + ); + } + + #[test] + fn a_pac_instruction_signs_its_register_and_a_branch_jumps_to_the_authenticated_target() { + let mut cpu = Cpu::default(); + cpu.system.sctlr_el1 = 1 << 31; + cpu.system.pac_keys[0] = KEY.lo; + cpu.system.pac_keys[1] = KEY.hi; + let ptr = 0x0000_0008_1234_5678; + cpu.x[0] = ptr; + cpu.x[1] = 0x2222; + run(&mut cpu, op(0xdac1_0020)); + assert_eq!(cpu.x[0], add_pac(ptr, 0x2222, KEY)); + cpu.x[2] = add_pac(0x4000, 0x2222, KEY); + cpu.x[3] = 0x2222; + cpu.pc = 0x1000; + run(&mut cpu, op(0xd71f_0800 | 2 << 5 | 3)); + assert_eq!(cpu.pc, 0x4000); + } +} diff --git a/src/emulator/src/aarch64/simd.rs b/src/emulator/src/aarch64/simd.rs index 3cced27..cc8bf10 100644 --- a/src/emulator/src/aarch64/simd.rs +++ b/src/emulator/src/aarch64/simd.rs @@ -5,432 +5,11 @@ //! wide arithmetic (`uaddl`, `umull`, ...), and the 64-bit scalar forms. //! //! Each register is a `u128`, each lane a bit field of it, so one lane loop serves -//! every element size. Decode only classifies a word with [`is_supported`]; the -//! JIT traps and the machine (or user mode) calls [`run`]. +//! every element size. Decoding lives in `volt_isa_aarch64::simd`; the JIT traps and +//! the machine (or user mode) calls [`run`]. use super::Cpu; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Permute { - Uzp1, - Uzp2, - Zip1, - Zip2, - Trn1, - Trn2, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Zero { - Gt, - Ge, - Le, - Lt, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Narrow { - Plain, - SignedSat, - UnsignedSat, - SignedToUnsignedSat, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Long { - Add, - AddWide, - Sub, - SubWide, - Abd, - Mul, - Mla, - Mls, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Scalar { - Add, - Sub, - Gt, - Hi, - Ge, - Hs, - Eq, - Tst, -} -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Op { - Permute { - kind: Permute, - size: u32, - q: bool, - rd: u8, - rn: u8, - rm: u8, - }, - /// `rev16`/`rev32`/`rev64`: reverse the lanes inside each group of `group` bits. - Reverse { - group: u32, - size: u32, - q: bool, - rd: u8, - rn: u8, - }, - Count { - q: bool, - rd: u8, - rn: u8, - }, - Not { - q: bool, - rd: u8, - rn: u8, - }, - ReverseBits { - q: bool, - rd: u8, - rn: u8, - }, - CompareZero { - kind: Zero, - size: u32, - q: bool, - rd: u8, - rn: u8, - }, - Abs { - negate: bool, - size: u32, - q: bool, - rd: u8, - rn: u8, - }, - /// `xtn`/`sqxtn`/`uqxtn`/`sqxtun`; `upper` is the `2` form (keeps the low half). - Narrow { - kind: Narrow, - size: u32, - upper: bool, - rd: u8, - rn: u8, - }, - /// `saddlp`/`uaddlp` and, with `accumulate`, `sadalp`/`uadalp`. - PairLong { - signed: bool, - accumulate: bool, - size: u32, - q: bool, - rd: u8, - rn: u8, - }, - Long { - kind: Long, - signed: bool, - size: u32, - upper: bool, - rd: u8, - rn: u8, - rm: u8, - }, - ScalarAddp { - rd: u8, - rn: u8, - }, - ScalarDup { - size: u32, - index: u32, - rd: u8, - rn: u8, - }, - ScalarThree { - kind: Scalar, - rd: u8, - rn: u8, - rm: u8, - }, - /// `shl`, and `sshr`/`ushr`, by an immediate. - ShiftImm { - right: bool, - signed: bool, - size: u32, - q: bool, - shift: u32, - rd: u8, - rn: u8, - }, - /// `sshll`/`ushll` (`sxtl`/`uxtl` at shift 0); `upper` is the `2` form. - ShiftLong { - signed: bool, - size: u32, - shift: u32, - upper: bool, - rd: u8, - rn: u8, - }, - /// `shrn`/`shrn2`. - ShiftNarrow { - size: u32, - shift: u32, - upper: bool, - rd: u8, - rn: u8, - }, -} - -fn parse(word: u32) -> Option { - let rd = (word & 31) as u8; - let rn = ((word >> 5) & 31) as u8; - let rm = ((word >> 16) & 31) as u8; - let q = word & 0x4000_0000 != 0; - let u = word & 0x2000_0000 != 0; - let size = (word >> 22) & 3; - // Permute: bit 21 clear, bits 11:10 = 10, bits 14:12 pick the operation. - if word & 0xbf20_8c00 == 0x0e00_0800 { - let kind = match (word >> 12) & 7 { - 1 => Permute::Uzp1, - 2 => Permute::Trn1, - 3 => Permute::Zip1, - 5 => Permute::Uzp2, - 6 => Permute::Trn2, - 7 => Permute::Zip2, - _ => return None, - }; - // 64-bit lanes need the full vector. - if size == 3 && !q { - return None; - } - return Some(Op::Permute { - kind, - size, - q, - rd, - rn, - rm, - }); - } - // Two-register miscellaneous: bits 21:17 = 10000, bits 11:10 = 10. - if word & 0x9f3e_0c00 == 0x0e20_0800 { - let opcode = (word >> 12) & 0x1f; - let wide_needs_q = size != 3 || q; - return match (u, opcode) { - (false, 0) if size < 3 => Some(Op::Reverse { - group: 64, - size, - q, - rd, - rn, - }), - (true, 0) if size < 2 => Some(Op::Reverse { - group: 32, - size, - q, - rd, - rn, - }), - (false, 1) if size == 0 => Some(Op::Reverse { - group: 16, - size, - q, - rd, - rn, - }), - (false | true, 2) if size < 3 => Some(Op::PairLong { - signed: !u, - accumulate: false, - size, - q, - rd, - rn, - }), - (false | true, 6) if size < 3 => Some(Op::PairLong { - signed: !u, - accumulate: true, - size, - q, - rd, - rn, - }), - (false, 5) if size == 0 => Some(Op::Count { q, rd, rn }), - (true, 5) if size == 0 => Some(Op::Not { q, rd, rn }), - (true, 5) if size == 1 => Some(Op::ReverseBits { q, rd, rn }), - (false, 8) if wide_needs_q => Some(Op::CompareZero { - kind: Zero::Gt, - size, - q, - rd, - rn, - }), - (true, 8) if wide_needs_q => Some(Op::CompareZero { - kind: Zero::Ge, - size, - q, - rd, - rn, - }), - (true, 9) if wide_needs_q => Some(Op::CompareZero { - kind: Zero::Le, - size, - q, - rd, - rn, - }), - (false, 10) if wide_needs_q => Some(Op::CompareZero { - kind: Zero::Lt, - size, - q, - rd, - rn, - }), - (false, 11) if wide_needs_q => Some(Op::Abs { - negate: false, - size, - q, - rd, - rn, - }), - (true, 11) if wide_needs_q => Some(Op::Abs { - negate: true, - size, - q, - rd, - rn, - }), - (false, 0x12) if size < 3 => Some(Op::Narrow { - kind: Narrow::Plain, - size, - upper: q, - rd, - rn, - }), - (true, 0x12) if size < 3 => Some(Op::Narrow { - kind: Narrow::SignedToUnsignedSat, - size, - upper: q, - rd, - rn, - }), - (false, 0x14) if size < 3 => Some(Op::Narrow { - kind: Narrow::SignedSat, - size, - upper: q, - rd, - rn, - }), - (true, 0x14) if size < 3 => Some(Op::Narrow { - kind: Narrow::UnsignedSat, - size, - upper: q, - rd, - rn, - }), - // `cmeq #0` (U=0, opcode 9) is `SimdCompareZero`. - _ => None, - }; - } - // Three registers different: bit 21 set, bits 11:10 = 00. - if word & 0x9f20_0c00 == 0x0e20_0000 && size < 3 { - let kind = match (word >> 12) & 15 { - 0 => Long::Add, - 1 => Long::AddWide, - 2 => Long::Sub, - 3 => Long::SubWide, - 7 => Long::Abd, - 8 => Long::Mla, - 10 => Long::Mls, - 12 => Long::Mul, - _ => return None, - }; - return Some(Op::Long { - kind, - signed: !u, - size, - upper: q, - rd, - rn, - rm, - }); - } - // Shift by immediate: `immh` (bits 22:19) gives the element size and, with - // `immb`, the shift. `immh == 0` is the modified-immediate class instead. - if word & 0x9f80_0400 == 0x0f00_0400 && (word >> 19) & 15 != 0 { - let immh = (word >> 19) & 15; - let immhb = (word >> 16) & 0x7f; - // The position of the top set bit of immh: 0 for bytes up to 3 for doublewords. - let size = 31 - immh.leading_zeros(); - let esize = 8 << size; - return match (u, (word >> 11) & 0x1f) { - (false, 0b01010) if size < 3 || q => Some(Op::ShiftImm { - right: false, - signed: false, - size, - q, - shift: immhb - esize, - rd, - rn, - }), - (_, 0b00000) if size < 3 || q => Some(Op::ShiftImm { - right: true, - signed: !u, - size, - q, - shift: 2 * esize - immhb, - rd, - rn, - }), - (_, 0b10100) if size < 3 => Some(Op::ShiftLong { - signed: !u, - size, - shift: immhb - esize, - upper: q, - rd, - rn, - }), - (false, 0b10000) if size < 3 => Some(Op::ShiftNarrow { - size, - shift: 2 * esize - immhb, - upper: q, - rd, - rn, - }), - _ => None, - }; - } - // Scalar pairwise `addp Dd, Vn.2D`. - if word & 0xffff_fc00 == 0x5ef1_b800 { - return Some(Op::ScalarAddp { rd, rn }); - } - // Scalar `dup`: `mov Bd/Hd/Sd/Dd, Vn.T[index]`. - if word & 0xffe0_fc00 == 0x5e00_0400 { - let imm5 = (word >> 16) & 0x1f; - if imm5 == 0 || imm5.trailing_zeros() > 3 { - return None; - } - let size = imm5.trailing_zeros(); - return Some(Op::ScalarDup { - size, - index: imm5 >> (size + 1), - rd, - rn, - }); - } - // Scalar three-same, 64-bit only. - if word & 0xdf20_0400 == 0x5e20_0400 && size == 3 { - let kind = match (u, (word >> 11) & 0x1f) { - (false, 0x10) => Scalar::Add, - (true, 0x10) => Scalar::Sub, - (false, 0x06) => Scalar::Gt, - (true, 0x06) => Scalar::Hi, - (false, 0x07) => Scalar::Ge, - (true, 0x07) => Scalar::Hs, - (true, 0x11) => Scalar::Eq, - (false, 0x11) => Scalar::Tst, - _ => return None, - }; - return Some(Op::ScalarThree { kind, rd, rn, rm }); - } - None -} - -/// Whether [`run`] implements `word`. -pub fn is_supported(word: u32) -> bool { - parse(word).is_some() -} +use volt_isa_aarch64::simd::{Long, Narrow, Op, Permute, Scalar, Zero}; fn read(cpu: &Cpu, reg: u8) -> u128 { let [lo, hi] = cpu.v[usize::from(reg)]; @@ -470,9 +49,8 @@ fn lanes(size: u32, q: bool) -> u32 { (if q { 128 } else { 64 }) / (8 << size) } -/// Run the instruction `word` on `cpu`. `word` must satisfy [`is_supported`]. -pub fn run(cpu: &mut Cpu, word: u32) { - let op = parse(word).expect("an unsupported SIMD word reached the executor"); +/// Run the decoded instruction `op` on `cpu`. +pub fn run(cpu: &mut Cpu, op: Op) { match op { Op::Permute { kind, @@ -789,6 +367,14 @@ pub fn run(cpu: &mut Cpu, word: u32) { mod tests { use super::*; + /// Decode `word` and run it, as the machine does for a trapped host word. + fn run(cpu: &mut Cpu, word: u32) { + super::run( + cpu, + volt_isa_aarch64::simd::decode(word).expect("a SIMD word"), + ); + } + /// A register whose lane `i` of `bits` bits is `f(i)`. fn pack(bits: u32, count: u32, f: impl Fn(u32) -> u128) -> [u64; 2] { let mut v = 0u128; @@ -1150,14 +736,4 @@ mod tests { run(&mut c, 0x0f3f8420); assert_eq!(c.v[0], [0x0000_0005_8000_0003, 0]); } - #[test] - fn classification_declines_what_it_does_not_implement() { - // cmeq #0 is `SimdCompareZero`; half-width lanes need Q for 64-bit lanes; - // `ins`, `umov` and the three-same group belong to other decoders. - assert!(!is_supported(0x0e209820)); - assert!(!is_supported(0x0ee0b820), "abs .1d is reserved"); - assert!(!is_supported(0x2ee21c20), "bif is three-same"); - assert!(!is_supported(0x4e083c20), "umov"); - assert!(is_supported(0x4e821820)); - } } diff --git a/src/emulator/src/aarch64/simd_struct.rs b/src/emulator/src/aarch64/simd_struct.rs index 9d1c9ab..59b7b5b 100644 --- a/src/emulator/src/aarch64/simd_struct.rs +++ b/src/emulator/src/aarch64/simd_struct.rs @@ -6,51 +6,7 @@ //! [`execute`] against its own memory, one element at a time, so each backend keeps //! its own translation, device and fault handling. -/// Which elements of the registers a structure access touches. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -#[repr(u8)] -pub enum Shape { - /// Whole vectors: `ld1 {v0.4s - v3.4s}` and the interleaved `ld2`..`ld4`. - #[default] - Multiple, - /// One element of each register: `ld1 {v0.s}[1]`. Other lanes are kept. - Lane, - /// One element broadcast to every lane: `ld1r`. Loads only. - Replicate, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -#[repr(C)] -pub struct StructDesc { - pub store: bool, - pub shape: Shape, - /// Registers touched, starting at `rt` and wrapping from v31 to v0. - pub registers: u8, - /// `ld2`..`ld4`: consecutive memory elements alternate between the registers. - /// Otherwise (`ld1`) each register is a contiguous block. - pub interleaved: bool, - /// Element size in bytes: 1, 2, 4 or 8. - pub esize: u8, - /// Element index of a [`Shape::Lane`] access. - pub lane: u8, - /// A 128-bit vector (otherwise 64-bit, with the upper half zeroed on loads). - pub q: bool, - pub rt: u8, -} - -impl StructDesc { - fn vector_bytes(&self) -> u8 { - if self.q { 16 } else { 8 } - } - /// Bytes of memory the access spans; the post-index immediate advances by this. - pub fn bytes(&self) -> u64 { - let registers = u64::from(self.registers); - match self.shape { - Shape::Multiple => registers * u64::from(self.vector_bytes()), - Shape::Lane | Shape::Replicate => registers * u64::from(self.esize), - } - } -} +use volt_isa_aarch64::simd_struct::{Shape, StructDesc}; fn mask(esize: u8) -> u64 { if esize >= 8 { diff --git a/src/emulator/src/aarch64/translate.rs b/src/emulator/src/aarch64/translate.rs index 9c4cedc..81ec534 100644 --- a/src/emulator/src/aarch64/translate.rs +++ b/src/emulator/src/aarch64/translate.rs @@ -26,23 +26,66 @@ pub struct Entry { pub generation: u64, pub ap: u8, pub executable: bool, - pub faulted: bool, } +/// The translation regime for the current EL. `split` regimes (EL1&0, and EL2 with +/// `HCR_EL2.E2H`, or EL0 under TGE with E2H) walk a TTBR0 and a TTBR1 range under the +/// `TCR_EL1` layout. EL2 without E2H has one TTBR0 range under `TCR_EL2`. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -struct Context { +struct Regime { sctlr: u64, tcr: u64, ttbr0: u64, ttbr1: u64, + split: bool, + /// Bit position of the physical-address size field: IPS (34:32) or PS (18:16). + ips_shift: u32, +} +impl Regime { + fn of(cpu: &Cpu) -> Self { + use super::cpu::{HCR_E2H, HCR_TGE}; + let hcr = cpu.system.hcr_el2; + let el2_regime = + cpu.system.el == 2 || (cpu.system.el == 0 && hcr & HCR_TGE != 0 && hcr & HCR_E2H != 0); + if !el2_regime { + return Self { + sctlr: cpu.system.sctlr_el1, + tcr: cpu.system.tcr_el1, + ttbr0: cpu.system.ttbr0_el1, + ttbr1: cpu.system.ttbr1_el1, + split: true, + ips_shift: 32, + }; + } + if hcr & HCR_E2H != 0 { + Self { + sctlr: cpu.system.sctlr_el2, + tcr: cpu.system.tcr_el2, + ttbr0: cpu.system.ttbr0_el2, + ttbr1: cpu.system.ttbr1_el2, + split: true, + ips_shift: 32, + } + } else { + Self { + sctlr: cpu.system.sctlr_el2, + tcr: cpu.system.tcr_el2, + ttbr0: cpu.system.ttbr0_el2, + ttbr1: 0, + split: false, + ips_shift: 16, + } + } + } +} +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct Context { + regime: Regime, el: u8, } impl Context { fn of(cpu: &Cpu) -> Self { Self { - sctlr: cpu.system.sctlr_el1, - tcr: cpu.system.tcr_el1, - ttbr0: cpu.system.ttbr0_el1, - ttbr1: cpu.system.ttbr1_el1, + regime: Regime::of(cpu), el: cpu.system.el, } } @@ -52,8 +95,6 @@ pub struct Tlb { pub entries: [Entry; 256], pub generation: u64, context: Option, - /// Some entry caches a translation fault. - has_faulted: bool, } impl Default for Tlb { fn default() -> Self { @@ -61,7 +102,6 @@ impl Default for Tlb { entries: [Entry::default(); 256], generation: 1, context: None, - has_faulted: false, } } } @@ -71,27 +111,10 @@ impl Tlb { } pub fn flush(&mut self) { self.generation = self.generation.wrapping_add(1); - self.has_faulted = false; if self.generation <= 1 { self.entries = [Entry::default(); 256]; } } - /// Drop cached translation faults, keeping every valid translation (and so - /// the generation that code and data fast paths are validated against). - /// A context-synchronization event (ISB) makes earlier page-table writes - /// visible to the walker, which can turn a fault into a mapping. It does - /// not invalidate valid translations; only TLBI does. - pub fn purge_faults(&mut self) { - if !self.has_faulted { - return; - } - self.has_faulted = false; - for entry in &mut self.entries { - if entry.faulted { - *entry = Entry::default(); - } - } - } fn slot(virtual_address: u64) -> usize { ((virtual_address >> 12) & 255) as usize } @@ -106,10 +129,11 @@ impl Tlb { } } fn enabled(cpu: &Cpu) -> Result { - if cpu.system.sctlr_el1 & 1 == 0 { + let regime = Regime::of(cpu); + if regime.sctlr & 1 == 0 { return Ok(false); } - if cpu.system.tcr_el1 & 63 == 0 { + if regime.tcr & 63 == 0 { return Err(TranslateError::MalformedTables); } Ok(true) @@ -118,7 +142,10 @@ pub fn physical_bits(cpu: &Cpu) -> Result { if !enabled(cpu)? { return Ok(64); } - Ok([32, 36, 40, 42, 48, 52, 56, 56][((cpu.system.tcr_el1 >> 32) & 7) as usize]) + let regime = Regime::of(cpu); + // IPS (TCR_EL1 bits 34:32) or PS (TCR_EL2 bits 18:16). + let field = (regime.tcr >> regime.ips_shift) & 7; + Ok([32, 36, 40, 42, 48, 52, 56, 56][field as usize]) } fn permitted(ap: u8, executable: bool, access: Access, el: u8) -> Result<(), TranslateError> { if (el == 0 && ap & 1 == 0) @@ -140,8 +167,11 @@ pub fn translate( if !enabled(cpu)? { return Ok(virtual_address); } - let t1sz = ((cpu.system.tcr_el1 >> 16) & 63) as u32; - if t1sz == 0 { + let regime = Regime::of(cpu); + let t1sz = ((regime.tcr >> 16) & 63) as u32; + // TTBR1's region size only means something while EPD1 leaves its walk enabled. + let upper_walk = regime.split && regime.tcr & (1 << 23) == 0; + if upper_walk && t1sz == 0 { return Err(TranslateError::MalformedTables); } if ((virtual_address >> 55) & 1 != 0 && virtual_address >> 48 != 0xffff) @@ -149,11 +179,11 @@ pub fn translate( { return Err(TranslateError::TranslationFault); } - let lower_bits = 64 - (cpu.system.tcr_el1 & 63) as u32; + let lower_bits = 64 - (regime.tcr & 63) as u32; let upper_bits = 64 - t1sz; let upper = if virtual_address >> lower_bits == 0 { false - } else if virtual_address >> upper_bits == u64::MAX >> upper_bits { + } else if upper_walk && virtual_address >> upper_bits == u64::MAX >> upper_bits { true } else { return Err(TranslateError::TranslationFault); @@ -166,9 +196,6 @@ pub fn translate( let slot = Tlb::slot(virtual_address); let entry = &tlb.entries[slot]; if entry.generation == tlb.generation && entry.virtual_address == page { - if entry.faulted { - return Err(TranslateError::TranslationFault); - } // A cached entry that denies the access is re-walked below: the // descriptor may have been upgraded since it was cached, which // hardware also resolves by retrying the walk. @@ -176,22 +203,10 @@ pub fn translate( return Ok(entry.physical | (virtual_address & PAGE_MASK)); } } - let found = match walk(cpu, memory, virtual_address, access, upper) { - Ok(found) => found, - Err(fault) => { - // Only access-independent failures may be cached as unmapped. - if fault != TranslateError::PermissionFault { - tlb.entries[slot] = Entry { - virtual_address: page, - generation: tlb.generation, - faulted: true, - ..Entry::default() - }; - tlb.has_faulted = true; - } - return Err(fault); - } - }; + // Faults are not cached. ARMv8 never caches an invalid descriptor, so a mapping + // installed later, without a TLBI or an ISB (as Linux's `ioremap` does), must be + // seen by the next access. + let found = walk(cpu, memory, virtual_address, access, upper)?; tlb.entries[slot] = Entry { virtual_address: page, physical: found.physical & !PAGE_MASK, @@ -207,8 +222,9 @@ fn walk( access: Access, upper: bool, ) -> Result { - let tcr = cpu.system.tcr_el1; - if (upper && tcr & (1 << 23) != 0) || (!upper && tcr & (1 << 7) != 0) { + let regime = Regime::of(cpu); + let tcr = regime.tcr; + if (upper && tcr & (1 << 23) != 0) || (!upper && regime.split && tcr & (1 << 7) != 0) { return Err(TranslateError::TranslationFault); } let granule = if upper { @@ -220,11 +236,7 @@ fn walk( return Err(TranslateError::MalformedTables); } // ASID is in TTBR bits 63:48, never part of a physical table address. - let root = if upper { - cpu.system.ttbr1_el1 - } else { - cpu.system.ttbr0_el1 - }; + let root = if upper { regime.ttbr1 } else { regime.ttbr0 }; let va_bits = 64 - if upper { (tcr >> 16) & 63 } else { tcr & 63 }; if va_bits > 48 || va_bits <= u64::from(granule) { return Err(TranslateError::MalformedTables); @@ -264,9 +276,6 @@ fn walk( let descriptor = u64::from_le_bytes(word); let kind = descriptor & 3; if kind == 3 && level != 0 { - if descriptor & 0x3f0 != 0 { - return Err(TranslateError::MalformedTables); - } no_user |= descriptor & (1 << 61) != 0; read_only |= descriptor & (1 << 62) != 0; pxn_table |= descriptor & (1 << 59) != 0; @@ -495,7 +504,7 @@ mod tests { assert_eq!(memory.reads.get(), reads, "same page must not walk again"); } #[test] - fn invalid_descriptors_and_malformed_tables_fail_closed() { + fn invalid_descriptors_fail_closed() { let cpu = cpu(); let mut memory = Memory::new(); let mut tlb = Tlb::new(); @@ -509,12 +518,24 @@ mod tests { translate(&mut tlb, &cpu, &mut memory, 0, Access::Read), Err(TranslateError::TranslationFault) ); - assert_eq!(memory.reads.get(), count); - tlb.flush(); - memory.put(0, 0x1013); + // A fault is not cached: the repeated access walks the tables again. + assert!(memory.reads.get() > count); + } + #[test] + fn table_descriptor_bits_eleven_to_two_are_ignored() { + // Table descriptors ignore bits 11:2 (ARM ARM D8.3.1). Set them on every table on the + // walk for address 0: the page at 0x4000_0000 must still resolve. + let cpu = cpu(); + let mut memory = Memory::new(); + let mut tlb = Tlb::new(); + memory.chain(0, 0, 12, 4, 0, 0x4000_0403); + for table in [0usize, 0x1000, 0x2000] { + let descriptor = memory.get(table); + memory.put(table as u64, descriptor | 0x3f0); + } assert_eq!( translate(&mut tlb, &cpu, &mut memory, 0, Access::Read), - Err(TranslateError::MalformedTables) + Ok(0x4000_0000) ); } #[test] @@ -729,4 +750,59 @@ mod tests { Err(TranslateError::PermissionFault) ); } + #[test] + fn el2_walks_only_ttbr0_el2_under_tcr_el2_and_has_no_upper_range() { + let mut cpu = Cpu::default(); + cpu.system.el = 2; + cpu.system.sctlr_el2 = 1; + // T0SZ = 16 (48-bit VAs), TG0 = 4 KiB, PS = 40 bits. + cpu.system.tcr_el2 = 16 | (2 << 16); + cpu.system.ttbr0_el2 = 0; + // The EL1&0 regime is disabled: EL2 translates from its own registers. + cpu.system.sctlr_el1 = 0; + let mut memory = Memory::new(); + memory.chain(0, 0, 12, 4, 0, 0x4000_0443); + let mut tlb = Tlb::new(); + assert_eq!( + translate(&mut tlb, &cpu, &mut memory, 0, Access::Read), + Ok(0x4000_0000) + ); + assert_eq!(physical_bits(&cpu), Ok(40)); + // Bit 55 set with an all-ones upper half: a TTBR1 address, which EL2 lacks. + assert_eq!( + translate( + &mut tlb, + &cpu, + &mut memory, + 0xffff_8000_0000_0000, + Access::Read + ), + Err(TranslateError::TranslationFault) + ); + } + #[test] + fn epd1_disables_the_upper_walk_so_its_t1sz_is_never_checked() { + // The tinted-boot firmware's TCR_EL1: T0SZ 25 (a 39-bit lower range), EPD1 set, + // and T1SZ 0, which is only meaningful while the TTBR1 walk is enabled. + let mut cpu = cpu(); + cpu.system.tcr_el1 = 25 | (1 << 23) | (2 << 30) | (4 << 32); + let mut memory = Memory::new(); + let mut tlb = Tlb::new(); + // A 1 GiB block at 0x4000_0000 in the TTBR0 root table, which sits at 0. + memory.chain(0, 0x4000_0000, 12, 3, 2, 0x4000_0401); + assert_eq!( + translate(&mut tlb, &cpu, &mut memory, 0x4000_0000, Access::Execute), + Ok(0x4000_0000) + ); + assert_eq!( + translate( + &mut tlb, + &cpu, + &mut memory, + 0xffff_8000_0000_0000, + Access::Read + ), + Err(TranslateError::TranslationFault) + ); + } } diff --git a/src/emulator/src/devices/aic.rs b/src/emulator/src/devices/aic.rs new file mode 100644 index 0000000..2a3e26e --- /dev/null +++ b/src/emulator/src/devices/aic.rs @@ -0,0 +1,470 @@ +//! Apple Interrupt Controller, AIC v1: the interrupt controller of the t8103 +//! (M1) SoC, as the Asahi Linux driver drives it on the `apple,t8103-aic` path. +//! +//! The guest sees 896 level-triggered hardware IRQ lines. Each IRQ has a mask +//! bit, a software trigger ORed with its line, and a target CPU. Each CPU has +//! one event register. Reading it returns the lowest-numbered IRQ that is +//! pending, unmasked and targeted at that CPU, and the read itself masks that +//! IRQ (auto-ack). The driver unmasks it again in its EOI handler. Because the +//! IRQ stays masked until then, a level line that is still high does not +//! re-enter the handler. +//! +//! IPIs on this SoC go through the fast-IPI system registers, which are not +//! part of this device. The driver still writes the MMIO IPI registers during +//! CPU bring-up (ACK, and MASK_SET for the SELF and OTHER bits). Those writes +//! are accepted: the mask registers store state and read it back. SEND, ACK, +//! and the per-CPU SET/CLR triggers are write-only no-ops, since no MMIO IPI is +//! ever sent on this path. +//! +//! Reset state: every IRQ is masked and targeted at CPU 0. The driver unmasks +//! an IRQ when it is requested. A guest never sees an IRQ that has not been +//! explicitly enabled. + +/// Mapping length of the AIC register window (the DT `reg` length). +pub const LEN: u64 = 0x8000; + +/// Hardware IRQ lines the guest sees. `AIC_INFO` reports this value, and the +/// driver's header comment documents 896 level-triggered hardware IRQs. Any IRQ +/// number at or above this is not a real pin: it has no effect on delivery, and +/// its register slots read as zero. +const NR_IRQ: usize = 896; +/// 32-bit words that cover `NR_IRQ` in each per-IRQ bitmap. +const IRQ_WORDS: usize = NR_IRQ / 32; + +const INFO: u64 = 0x0004; +const WHOAMI: u64 = 0x2000; +const EVENT: u64 = 0x2004; +const IPI_SEND: u64 = 0x2008; +const IPI_ACK: u64 = 0x200c; +const IPI_MASK_SET: u64 = 0x2024; +const IPI_MASK_CLR: u64 = 0x2028; + +/// `TARGET_CPU`: one word per IRQ at `0x3000 + 4 * irq`. The word holds the CPU +/// bit(s) that receive the IRQ. The driver writes `BIT(cpu)`. +const TARGET_CPU: u64 = 0x3000; + +/// Per-IRQ bitmap register blocks. The driver lays them out in this order, and +/// each block is 32 words (128 bytes) long, one bit per IRQ. +const SW_SET: u64 = 0x4000; +#[cfg(test)] +const SW_CLR: u64 = 0x4080; +#[cfg(test)] +const MASK_SET: u64 = 0x4100; +#[cfg(test)] +const MASK_CLR: u64 = 0x4180; +const HW_STATE: u64 = 0x4200; +const BITMAP_STRIDE: u64 = 0x80; + +/// Explicit per-CPU IPI views: `0x5000 + cpu * 0x80`, with the registers at +/// fixed offsets within each 128-byte block. +const CPU_IPI_BASE: u64 = 0x5000; +const CPU_IPI_STRIDE: u64 = 0x80; +const CPU_IPI_SET: u64 = 0x08; +const CPU_IPI_CLR: u64 = 0x0c; +const CPU_IPI_MASK_SET: u64 = 0x24; +const CPU_IPI_MASK_CLR: u64 = 0x28; + +/// Event word for an IRQ: type `IRQ` (1) in bits 23:16, IRQ number in 15:0. +const EVENT_TYPE_IRQ: u32 = 1 << 16; + +/// Which per-IRQ bitmap a register offset selects. +#[derive(Clone, Copy)] +enum Bitmap { + SwSet, + SwClr, + MaskSet, + MaskClr, + HwState, +} + +/// MMIO IPI registers, both the default view and the explicit per-CPU view. +#[derive(Clone, Copy)] +enum IpiReg { + Send, + Ack, + Set, + Clr, + MaskSet, + MaskClr, +} + +/// A decoded, word-aligned register offset. +#[derive(Clone, Copy)] +enum Reg { + Info, + WhoAmI, + Event, + Target(usize), + Bitmap(Bitmap, usize), + Ipi { cpu: u32, reg: IpiReg }, + Reserved, +} + +/// Apple Interrupt Controller (AIC v1) state for `cpus` CPUs. +pub struct Aic { + cpus: u32, + /// Hardware line state, one bit per IRQ. + hw: [u32; IRQ_WORDS], + /// Software trigger state (`SW_SET`/`SW_CLR`), one bit per IRQ. + sw: [u32; IRQ_WORDS], + /// Mask state (`MASK_SET`/`MASK_CLR`), one bit per IRQ. A set bit masks. + mask: [u32; IRQ_WORDS], + /// CPU routing per IRQ (`TARGET_CPU`). Only the first `NR_IRQ` entries are + /// meaningful. + target: [u32; NR_IRQ], + /// Per-CPU IPI mask state, indexed by CPU. Only the first `cpus` entries + /// are reachable. + ipi_mask: [u32; 32], +} + +/// Mask of the low `n` bytes (`n` in 1..=4) of a 32-bit word. +fn byte_mask(n: u32) -> u32 { + if n >= 4 { + u32::MAX + } else { + (1u32 << (n * 8)) - 1 + } +} + +impl Aic { + /// `cpus` = number of CPUs wired to the IRQ pins (affects per-CPU IPI registers and target masks). + /// + /// Panics if `cpus` is outside `1..=32`, since target registers are 32-bit CPU masks. + pub fn new(cpus: u32) -> Self { + assert!((1..=32).contains(&cpus), "AIC routes to 1..=32 CPUs"); + Self { + cpus, + hw: [0; IRQ_WORDS], + sw: [0; IRQ_WORDS], + mask: [u32::MAX; IRQ_WORDS], + target: [1; NR_IRQ], + ipi_mask: [u32::MAX; 32], + } + } + + /// Drive hardware IRQ `irq` (level-triggered source line). + /// + /// IRQ numbers at or above `NR_IRQ` have no pin and are ignored. + pub fn set_irq(&mut self, irq: u32, asserted: bool) { + let irq = irq as usize; + if irq >= NR_IRQ { + return; + } + let (w, bit) = (irq / 32, 1u32 << (irq % 32)); + if asserted { + self.hw[w] |= bit; + } else { + self.hw[w] &= !bit; + } + } + + /// Whether the IRQ pin of CPU `cpu` is asserted right now (an enabled, unmasked, targeted source is pending). + pub fn irq_pending(&self, cpu: u32) -> bool { + self.first_pending(cpu).is_some() + } + + /// MMIO read at `offset` from the AIC base, performed by CPU `cpu` (AIC_WHOAMI and AIC_EVENT are per reading CPU). Takes &mut self because reading AIC_EVENT has side effects. + /// + /// `size` is 1, 2, 4 or 8 bytes. Accesses that cross a 32-bit register are + /// split per register. Unmapped offsets read as zero. + pub fn read(&mut self, cpu: u32, offset: u64, size: u8) -> u64 { + let size = u64::from(size.min(8)); + let mut value = 0u64; + let mut done = 0u64; + while done < size { + let addr = offset.wrapping_add(done); + let lane = (addr & 3) as u32; + let n = (4 - u64::from(lane)).min(size - done) as u32; + let word = self.read_word(cpu, addr & !3); + let bytes = u64::from((word >> (lane * 8)) & byte_mask(n)); + value |= bytes << (done * 8); + done += u64::from(n); + } + value + } + + /// MMIO write. Only the low `size` bytes of `value` are written. Writes to + /// set/clear registers only affect the bits they name. Writes to read-only + /// or unmapped offsets are ignored. + pub fn write(&mut self, cpu: u32, offset: u64, size: u8, value: u64) { + let size = u64::from(size.min(8)); + let mut done = 0u64; + while done < size { + let addr = offset.wrapping_add(done); + let lane = (addr & 3) as u32; + let n = (4 - u64::from(lane)).min(size - done) as u32; + let lanes = byte_mask(n) << (lane * 8); + let bits = (((value >> (done * 8)) as u32) << (lane * 8)) & lanes; + self.write_word(cpu, addr & !3, bits, lanes); + done += u64::from(n); + } + } + + /// Lowest IRQ that is pending, unmasked, and targeted at `cpu`. A CPU index + /// of 32 or more has no routing bit, so it never matches. + fn first_pending(&self, cpu: u32) -> Option { + let bit = 1u32.checked_shl(cpu).unwrap_or(0); + for w in 0..IRQ_WORDS { + let mut candidates = (self.hw[w] | self.sw[w]) & !self.mask[w]; + while candidates != 0 { + let irq = w * 32 + candidates.trailing_zeros() as usize; + if self.target[irq] & bit != 0 { + return Some(irq); + } + candidates &= candidates - 1; + } + } + None + } + + /// Reading AIC_EVENT: deliver the lowest pending IRQ for `cpu` and mask it + /// (auto-ack). Returns 0 when nothing is pending. + fn claim(&mut self, cpu: u32) -> u32 { + let Some(irq) = self.first_pending(cpu) else { + return 0; + }; + self.mask[irq / 32] |= 1 << (irq % 32); + EVENT_TYPE_IRQ | irq as u32 + } + + fn own_ipi(&self, cpu: u32, reg: IpiReg) -> Reg { + if cpu < self.cpus { + Reg::Ipi { cpu, reg } + } else { + Reg::Reserved + } + } + + fn decode(&self, offset: u64, cpu: u32) -> Reg { + match offset { + INFO => Reg::Info, + WHOAMI => Reg::WhoAmI, + EVENT => Reg::Event, + IPI_SEND => self.own_ipi(cpu, IpiReg::Send), + IPI_ACK => self.own_ipi(cpu, IpiReg::Ack), + IPI_MASK_SET => self.own_ipi(cpu, IpiReg::MaskSet), + IPI_MASK_CLR => self.own_ipi(cpu, IpiReg::MaskClr), + o if (TARGET_CPU..SW_SET).contains(&o) => { + let irq = ((o - TARGET_CPU) / 4) as usize; + if irq < NR_IRQ { + Reg::Target(irq) + } else { + Reg::Reserved + } + } + o if (SW_SET..HW_STATE + BITMAP_STRIDE).contains(&o) => { + let block = (o - SW_SET) / BITMAP_STRIDE; + let word = (((o - SW_SET) % BITMAP_STRIDE) / 4) as usize; + let which = match block { + 0 => Bitmap::SwSet, + 1 => Bitmap::SwClr, + 2 => Bitmap::MaskSet, + 3 => Bitmap::MaskClr, + _ => Bitmap::HwState, + }; + if word < IRQ_WORDS { + Reg::Bitmap(which, word) + } else { + Reg::Reserved + } + } + o if (CPU_IPI_BASE..CPU_IPI_BASE + CPU_IPI_STRIDE * u64::from(self.cpus)) + .contains(&o) => + { + let rel = o - CPU_IPI_BASE; + let reg = match rel % CPU_IPI_STRIDE { + CPU_IPI_SET => IpiReg::Set, + CPU_IPI_CLR => IpiReg::Clr, + CPU_IPI_MASK_SET => IpiReg::MaskSet, + CPU_IPI_MASK_CLR => IpiReg::MaskClr, + _ => return Reg::Reserved, + }; + Reg::Ipi { + cpu: (rel / CPU_IPI_STRIDE) as u32, + reg, + } + } + _ => Reg::Reserved, + } + } + + /// Read the 32-bit register at word-aligned `offset`. + fn read_word(&mut self, cpu: u32, offset: u64) -> u32 { + match self.decode(offset, cpu) { + Reg::Info => NR_IRQ as u32, + Reg::WhoAmI => cpu, + Reg::Event => self.claim(cpu), + Reg::Target(irq) => self.target[irq], + Reg::Bitmap(Bitmap::SwSet | Bitmap::SwClr, w) => self.sw[w], + Reg::Bitmap(Bitmap::MaskSet | Bitmap::MaskClr, w) => self.mask[w], + Reg::Bitmap(Bitmap::HwState, w) => self.hw[w], + Reg::Ipi { + cpu: c, + reg: IpiReg::MaskSet | IpiReg::MaskClr, + } => self.ipi_mask[c as usize], + Reg::Ipi { .. } | Reg::Reserved => 0, + } + } + + /// Write the lanes in `lanes` of the 32-bit register at word-aligned + /// `offset`. `bits` holds the written value already shifted into those + /// lanes and zero elsewhere. + fn write_word(&mut self, cpu: u32, offset: u64, bits: u32, lanes: u32) { + match self.decode(offset, cpu) { + Reg::Target(irq) => self.target[irq] = (self.target[irq] & !lanes) | bits, + Reg::Bitmap(Bitmap::SwSet, w) => self.sw[w] |= bits, + Reg::Bitmap(Bitmap::SwClr, w) => self.sw[w] &= !bits, + Reg::Bitmap(Bitmap::MaskSet, w) => self.mask[w] |= bits, + Reg::Bitmap(Bitmap::MaskClr, w) => self.mask[w] &= !bits, + Reg::Ipi { + cpu: c, + reg: IpiReg::MaskSet, + } => self.ipi_mask[c as usize] |= bits, + Reg::Ipi { + cpu: c, + reg: IpiReg::MaskClr, + } => self.ipi_mask[c as usize] &= !bits, + // Read-only registers, write-only IPI triggers, and reserved + // offsets: accepted and ignored. + Reg::Info + | Reg::WhoAmI + | Reg::Event + | Reg::Bitmap(Bitmap::HwState, _) + | Reg::Ipi { .. } + | Reg::Reserved => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Address of the MASK_CLR bit for `irq`. + fn mask_clr(irq: u32) -> u64 { + MASK_CLR + 4 * u64::from(irq / 32) + } + + /// Clear the mask of `irq` on CPU 0, as the driver does when it enables an IRQ. + fn unmask(aic: &mut Aic, irq: u32) { + aic.write(0, mask_clr(irq), 4, 1u64 << (irq % 32)); + } + + fn event(aic: &mut Aic, cpu: u32) -> u64 { + aic.read(cpu, EVENT, 4) + } + + #[test] + fn event_returns_lowest_pending_irq_and_auto_masks() { + let mut aic = Aic::new(1); + unmask(&mut aic, 40); + unmask(&mut aic, 5); + aic.set_irq(40, true); + aic.set_irq(5, true); + + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 5)); + // IRQ 5 was auto-masked by the read, so IRQ 40 is next. + assert!(aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 40)); + assert!(!aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), 0); + } + + #[test] + fn masked_irq_does_not_assert_pin_until_unmasked() { + let mut aic = Aic::new(1); + aic.set_irq(3, true); + assert!(!aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), 0); + + unmask(&mut aic, 3); + assert!(aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 3)); + } + + #[test] + fn reasserting_while_auto_masked_does_not_raise_pin() { + let mut aic = Aic::new(1); + unmask(&mut aic, 3); + aic.set_irq(3, true); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 3)); + + // The line drops and rises again while the IRQ is auto-masked. + aic.set_irq(3, false); + aic.set_irq(3, true); + assert!(!aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), 0); + } + + #[test] + fn mask_clr_reenables_and_pin_rises_again() { + let mut aic = Aic::new(1); + unmask(&mut aic, 3); + aic.set_irq(3, true); + event(&mut aic, 0); + assert!(!aic.irq_pending(0)); + + // The driver's EOI unmasks the level-triggered source, which is still high. + unmask(&mut aic, 3); + assert!(aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 3)); + } + + #[test] + fn target_register_routes_irq_away_from_reading_cpu() { + let mut aic = Aic::new(2); + unmask(&mut aic, 7); + aic.set_irq(7, true); + // Reset routes every IRQ to CPU 0. + assert!(aic.irq_pending(0)); + + // The driver writes BIT(1) to route IRQ 7 to CPU 1. + aic.write(0, TARGET_CPU + 4 * 7, 4, 1 << 1); + assert!(!aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), 0); + assert!(aic.irq_pending(1)); + assert_eq!(event(&mut aic, 1), u64::from(EVENT_TYPE_IRQ | 7)); + } + + #[test] + fn software_trigger_asserts_without_hardware_line() { + let mut aic = Aic::new(1); + unmask(&mut aic, 40); + // IRQ 40 is bit 8 of the second word of each bitmap block. + aic.write(0, SW_SET + 4, 4, 1 << 8); + assert!(aic.irq_pending(0)); + assert_eq!(event(&mut aic, 0), u64::from(EVENT_TYPE_IRQ | 40)); + + aic.write(0, SW_SET + 4, 4, 1 << 8); + aic.write(0, SW_CLR + 4, 4, 1 << 8); + assert!(!aic.irq_pending(0)); + } + + #[test] + fn whoami_reports_the_reading_cpu() { + let mut aic = Aic::new(2); + assert_eq!(aic.read(0, WHOAMI, 4), 0); + assert_eq!(aic.read(1, WHOAMI, 4), 1); + } + + #[test] + fn unknown_offsets_read_zero_and_ignore_writes() { + let mut aic = Aic::new(1); + // Reserved gap, AIC_CONFIG (not used by the driver on this path), a + // reserved word in the mask block, a target slot past the last IRQ, a + // bitmap block past HW_STATE, and a per-CPU view for a CPU that is absent. + let offsets = [ + 0x0000, + 0x0010, + 0x2010, + MASK_SET + 4 * 28, + TARGET_CPU + 4 * NR_IRQ as u64, + HW_STATE + BITMAP_STRIDE, + CPU_IPI_BASE + CPU_IPI_STRIDE, + ]; + for offset in offsets { + aic.write(0, offset, 4, u64::from(u32::MAX)); + assert_eq!(aic.read(0, offset, 4), 0, "offset {offset:#x}"); + } + } +} diff --git a/src/emulator/src/devices/fw_cfg.rs b/src/emulator/src/devices/fw_cfg.rs new file mode 100644 index 0000000..460a446 --- /dev/null +++ b/src/emulator/src/devices/fw_cfg.rs @@ -0,0 +1,98 @@ +//! QEMU fw_cfg's traditional MMIO interface. DMA is not advertised. +use super::bus::Device; + +pub const BASE: u64 = 0x0902_0000; +// Exclude the DMA address register at offset 0x10 from the advertised window. +pub const LEN: u64 = 0x10; + +pub struct FwCfg { + selector: u16, + offset: usize, + ram_size: [u8; 8], +} + +impl FwCfg { + pub fn new(ram_size: u64) -> Self { + Self { + selector: 0, + offset: 0, + ram_size: ram_size.to_le_bytes(), + } + } + + fn item(&self) -> &[u8] { + match self.selector { + 0x00 => b"QEMU", + 0x01 => &[1, 0, 0, 0], // Traditional interface only; no DMA. + 0x03 => &self.ram_size, + 0x05 | 0x0f => &[1, 0], // Number and maximum number of CPUs. + 0x19 => &[0, 0, 0, 0], // Empty file directory (big-endian count). + _ => &[], + } + } +} + +impl Device for FwCfg { + fn read(&mut self, offset: u64, size: u8) -> u64 { + if offset != 0 || !matches!(size, 1 | 2 | 4 | 8) { + return 0; + } + let item = self.item(); + let mut value = 0; + for byte in 0..usize::from(size) { + let at = self.offset.saturating_add(byte); + value |= u64::from(item.get(at).copied().unwrap_or(0)) << (byte * 8); + } + self.offset = self.offset.saturating_add(usize::from(size)); + value + } + + fn write(&mut self, offset: u64, size: u8, value: u64) { + if offset == 8 && size == 2 { + // MMIO selectors are big-endian; Device values are little-endian. + self.selector = (value as u16).swap_bytes() & !0x4000; + self.offset = 0; + } + // The data interface is read-only, and there is no DMA register. + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stream_preserves_bytes_across_widths_and_reselection() { + let mut device = FwCfg::new(0x0102_0304_0506_0708); + assert_eq!(device.read(0, 1), u64::from(b'Q')); + assert_eq!(device.read(0, 2), u64::from(u16::from_le_bytes(*b"EM"))); + assert_eq!(device.read(0, 4), u64::from(b'U')); + assert_eq!(device.read(0, 8), 0); + device.write(8, 2, 0); + assert_eq!(device.read(0, 4), u64::from(u32::from_le_bytes(*b"QEMU"))); + device.write(8, 2, 0x0300); + assert_eq!(device.read(0, 8), 0x0102_0304_0506_0708); + device.write(8, 2, 0x0100); + assert_eq!(device.read(0, 4), 1); + // The obsolete write-channel flag does not select a different item. + device.write(8, 2, 0x0040); + device.write(0, 4, 0); + assert_eq!(device.read(0, 4), u64::from(u32::from_le_bytes(*b"QEMU"))); + } + + #[test] + fn absent_items_and_unsupported_registers_do_not_change_the_stream() { + let mut device = FwCfg::new(128 << 20); + device.write(8, 2, 0x1900); + assert_eq!(device.read(0, 4), 0); + device.write(8, 2, 0xffff); + assert_eq!(device.read(0, 8), 0); + device.write(8, 2, 0); + device.write(0, 4, 0); + device.write(8, 1, 1); + device.write(16, 8, 0); + assert_eq!(device.read(8, 2), 0); + assert_eq!(device.read(16, 8), 0); + assert_eq!(device.read(0, 4), u64::from(u32::from_le_bytes(*b"QEMU"))); + } +} diff --git a/src/emulator/src/devices/gicv3.rs b/src/emulator/src/devices/gicv3.rs index 8611ca6..c1c521b 100644 --- a/src/emulator/src/devices/gicv3.rs +++ b/src/emulator/src/devices/gicv3.rs @@ -50,6 +50,17 @@ pub struct Gicv3 { enabled: u32, pending: u32, priority: [u8; 32], + /// `ITLinesNumber` as the distributor reports it. Zero is a board with 32 INTIDs and no + /// shared interrupts; firmware that registers interrupts sized by `arm-gic` needs more. + it_lines: u8, +} + +impl Gicv3 { + /// A distributor with `it_lines` interrupt lines (see the field). + pub fn with_it_lines(mut self, it_lines: u8) -> Self { + self.it_lines = it_lines; + self + } } impl Default for Gicv3 { @@ -62,6 +73,7 @@ impl Default for Gicv3 { enabled: 0, pending: 0, priority: [0; 32], + it_lines: 0, } } } @@ -82,8 +94,8 @@ impl Gicv3 { // All of these are 32-bit registers. let register = match offset & !3 { GICD_CTLR => u64::from(self.distributor_ctlr), - // `ITLinesNumber` 0 (32 INTIDs), one PE, 16 ID bits. - GICD_TYPER => 15 << 19, + // `ITLinesNumber` (`it_lines`) gives 32 * (it_lines + 1) INTIDs, one PE, 16 ID bits. + GICD_TYPER => (15 << 19) | u64::from(self.it_lines), PIDR2 => PIDR2_GICV3, _ => 0, }; diff --git a/src/emulator/src/devices/mod.rs b/src/emulator/src/devices/mod.rs index 72ce939..eb520bd 100644 --- a/src/emulator/src/devices/mod.rs +++ b/src/emulator/src/devices/mod.rs @@ -1,11 +1,21 @@ +pub mod aic; pub mod bus; +pub mod fw_cfg; pub mod gicv2; pub mod gicv3; +pub mod nor_flash; +pub mod pci; pub mod pl011; +pub mod pl031; +pub mod pmgr; +pub mod s5l_uart; pub mod virtio_blk; +pub mod virtio_pci; pub use bus::{Bus, BusDevice, Device}; pub use gicv2::Gicv2; pub use gicv3::Gicv3; +pub use pci::PciHost; pub use pl011::{LEN as PL011_LEN, Pl011}; pub use virtio_blk::{BlockBackend, VirtioBlock, VirtioIo}; +pub use virtio_pci::VirtioBlockPci; diff --git a/src/emulator/src/devices/nor_flash.rs b/src/emulator/src/devices/nor_flash.rs new file mode 100644 index 0000000..46ece2a --- /dev/null +++ b/src/emulator/src/devices/nor_flash.rs @@ -0,0 +1,611 @@ +//! Volatile variable flash: the second QEMU virt bank, not the firmware ROM. +//! +//! The bank is two x16 Intel-command-set chips on a little-endian x32 bus. +//! Operations complete synchronously; storage and the write buffer are allocated +//! only when the device is constructed. Command decoding follows QEMU's bank-wide +//! low-byte commands, with status and identification repeated in both x16 lanes. + +use alloc::vec; +use alloc::vec::Vec; + +use super::bus::Device; + +pub const BASE: u64 = 0x0400_0000; +pub const LEN: u64 = 64 * 1024 * 1024; +pub const BLOCK_SIZE: u64 = 256 * 1024; + +const BLOCK_COUNT: usize = (LEN / BLOCK_SIZE) as usize; +// QEMU pflash_cfi01 advertises 2 KiB per chip, hence 4 KiB for the bank. +// EDK2's P30 driver uses a smaller, compatible 128-byte buffer. +const BUFFER_SIZE: usize = 4096; +const BUFFER_WORDS: usize = BUFFER_SIZE / 4; +const READY: u8 = 0x80; +const ERASE_ERROR: u8 = 0x20; +const PROGRAM_ERROR: u8 = 0x10; +const BLOCK_LOCKED: u8 = 0x02; +const SEQUENCE_ERROR: u8 = ERASE_ERROR | PROGRAM_ERROR; + +#[derive(Clone, Copy)] +enum ReadMode { + Array, + Status, + Id, + Cfi, +} + +#[derive(Clone, Copy)] +enum WriteState { + Idle, + Program, + Erase { + block: usize, + }, + Lock { + block: usize, + }, + BufferCount { + block: usize, + }, + BufferData { + block: usize, + remaining: usize, + window: Option, + }, + BufferConfirm, +} + +#[derive(Clone, Copy)] +struct BufferedWord { + offset: u32, + value: u32, +} + +pub struct NorFlash { + storage: Vec, + locked: [bool; BLOCK_COUNT], + mode: ReadMode, + state: WriteState, + status: u8, + buffer: [BufferedWord; BUFFER_WORDS], + buffered_words: usize, +} + +impl NorFlash { + /// Construct an erased, unlocked variable bank. Its contents are not persisted. + pub fn new() -> Self { + Self { + storage: vec![0xff; LEN as usize], + locked: [false; BLOCK_COUNT], + mode: ReadMode::Array, + state: WriteState::Idle, + status: READY, + buffer: [BufferedWord { + offset: 0, + value: 0, + }; BUFFER_WORDS], + buffered_words: 0, + } + } + + fn valid_access(offset: u64, size: u8) -> bool { + matches!(size, 1 | 2 | 4 | 8) + && offset + .checked_add(u64::from(size)) + .is_some_and(|end| end <= LEN) + } + + fn block(offset: usize) -> usize { + offset / BLOCK_SIZE as usize + } + + fn read_array(&mut self) { + self.mode = ReadMode::Array; + self.state = WriteState::Idle; + self.buffered_words = 0; + } + + fn fail(&mut self, bits: u8) { + self.status |= READY | bits; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + self.buffered_words = 0; + } + + fn id(&self, offset: usize) -> u16 { + // QEMU virt IDs, from hw/arm/virt.c. ID address 2 is block lock status. + match (offset / 4) & 0xff { + 0 => 0x89, + 1 => 0x18, + 2 => u16::from(self.locked[Self::block(offset)]), + _ => 0, + } + } + + fn cfi(address: usize) -> u8 { + // Geometry and Intel PRI table from QEMU hw/block/pflash_cfi01.c. + // The CFI geometry is per chip: 32 MiB, 256 blocks of 128 KiB each. + match address { + 0x10 => b'Q', + 0x11 => b'R', + 0x12 => b'Y', + 0x13 => 1, + 0x15 => 0x31, + 0x1b => 0x45, + 0x1c => 0x55, + 0x1f | 0x20 => 7, + 0x21 => 0x0a, + 0x23..=0x25 => 4, + 0x27 => 25, + 0x28 => 2, + 0x2a => 11, + 0x2c => 1, + 0x2d => 0xff, + 0x30 => 2, + 0x31 => b'P', + 0x32 => b'R', + 0x33 => b'I', + 0x34 => b'1', + 0x35 => b'0', + 0x3f => 1, + _ => 0, + } + } + + fn program(&mut self, offset: usize, size: u8, value: u64) { + if self.locked[Self::block(offset)] + || self.locked[Self::block(offset + usize::from(size) - 1)] + { + self.status |= PROGRAM_ERROR | BLOCK_LOCKED; + return; + } + for byte in 0..usize::from(size) { + let new = (value >> (byte * 8)) as u8; + let old = self.storage[offset + byte]; + if new & !old != 0 { + self.status |= PROGRAM_ERROR; + } + // Programming can clear bits only; even erroneous writes never erase. + self.storage[offset + byte] = old & new; + } + } + + fn command(&mut self, offset: usize, command: u8) { + self.state = WriteState::Idle; + match command { + // 0x00 and 0xf0 are QEMU's historical read-array/probe aliases. + 0x00 | 0xff | 0xf0 => self.read_array(), + 0x10 | 0x40 => { + self.mode = ReadMode::Status; + self.state = WriteState::Program; + } + 0x20 => { + self.mode = ReadMode::Status; + self.state = WriteState::Erase { + block: Self::block(offset), + }; + } + 0x50 => { + // Clear sticky errors, not the ready bit. Like QEMU, return to array. + self.status = READY; + self.read_array(); + } + 0x60 => { + self.mode = ReadMode::Status; + self.state = WriteState::Lock { + block: Self::block(offset), + }; + } + 0x70 => self.mode = ReadMode::Status, + 0x90 => self.mode = ReadMode::Id, + 0x98 => self.mode = ReadMode::Cfi, + 0xe8 => { + self.mode = ReadMode::Status; + self.buffered_words = 0; + self.state = WriteState::BufferCount { + block: Self::block(offset), + }; + } + _ => self.fail(SEQUENCE_ERROR), + } + } +} + +impl Default for NorFlash { + fn default() -> Self { + Self::new() + } +} + +impl Device for NorFlash { + fn read(&mut self, offset: u64, size: u8) -> u64 { + if !Self::valid_access(offset, size) { + return 0; + } + let offset = offset as usize; + let mut value = 0; + for byte in 0..usize::from(size) { + let at = offset + byte; + let data = match self.mode { + ReadMode::Array => self.storage[at], + ReadMode::Status => { + if at & 1 == 0 { + self.status + } else { + 0 + } + } + ReadMode::Id => (self.id(at) >> ((at & 1) * 8)) as u8, + ReadMode::Cfi => { + if at & 1 == 0 { + Self::cfi(at / 4) + } else { + 0 + } + } + }; + value |= u64::from(data) << (byte * 8); + } + value + } + + fn write(&mut self, offset: u64, size: u8, value: u64) { + // A bank write is at most 32 bits. 64-bit array reads are supported, but + // 64-bit writes are not a single command/data cycle on the x32 bus. + if !Self::valid_access(offset, size) || size == 8 { + self.fail(SEQUENCE_ERROR); + return; + } + let offset = offset as usize; + let command = value as u8; + match self.state { + WriteState::Idle => self.command(offset, command), + WriteState::Program => { + // This cycle is data, even if its low byte resembles a command. + self.program(offset, size, value); + self.status |= READY; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + } + WriteState::Erase { block } => { + if command == 0xff { + self.read_array(); + } else if command != 0xd0 || Self::block(offset) != block { + self.fail(SEQUENCE_ERROR); + } else { + if self.locked[block] { + self.status |= ERASE_ERROR | BLOCK_LOCKED; + } else { + let start = block * BLOCK_SIZE as usize; + self.storage[start..start + BLOCK_SIZE as usize].fill(0xff); + } + self.status |= READY; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + } + } + WriteState::Lock { block } => { + if command == 0xff { + self.read_array(); + } else if !matches!(command, 0x01 | 0xd0) || Self::block(offset) != block { + self.fail(SEQUENCE_ERROR); + } else { + self.locked[block] = command == 0x01; + self.status |= READY; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + } + } + WriteState::BufferCount { block } => { + // Counts are per x16 chip and zero means one bank word. + let words = usize::from(value as u16) + 1; + if size != 4 + || offset & 3 != 0 + || Self::block(offset) != block + || words > BUFFER_WORDS + { + self.fail(SEQUENCE_ERROR); + } else { + self.state = WriteState::BufferData { + block, + remaining: words, + window: None, + }; + } + } + WriteState::BufferData { + block, + remaining, + window, + } => { + let data_window = offset & !(BUFFER_SIZE - 1); + if size != 4 + || offset & 3 != 0 + || Self::block(offset) != block + || window.is_some_and(|start| start != data_window) + { + self.fail(SEQUENCE_ERROR); + return; + } + self.buffer[self.buffered_words] = BufferedWord { + offset: offset as u32, + value: value as u32, + }; + self.buffered_words += 1; + self.state = if remaining == 1 { + WriteState::BufferConfirm + } else { + WriteState::BufferData { + block, + remaining: remaining - 1, + window: Some(data_window), + } + }; + } + WriteState::BufferConfirm => { + if command == 0xff { + self.read_array(); + } else if command != 0xd0 { + self.fail(SEQUENCE_ERROR); + } else { + // EDK2 confirms at DeviceBaseAddress, not the target block. + // Do not restrict the confirmation address to the data window. + for index in 0..self.buffered_words { + let word = self.buffer[index]; + self.program(word.offset as usize, 4, u64::from(word.value)); + } + self.buffered_words = 0; + self.status |= READY; + self.mode = ReadMode::Status; + self.state = WriteState::Idle; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn command(flash: &mut NorFlash, offset: u64, command: u16) { + flash.write(offset, 4, u64::from(command) | (u64::from(command) << 16)); + } + + fn status(flash: &mut NorFlash) -> u64 { + command(flash, 0, 0x70); + flash.read(0, 4) + } + + fn program(flash: &mut NorFlash, offset: u64, value: u32) { + command(flash, offset, 0x40); + flash.write(offset, 4, u64::from(value)); + } + + #[test] + fn erased_array_and_programming_preserve_little_endian_accesses() { + let mut flash = NorFlash::new(); + assert_eq!(flash.read(0, 8), u64::MAX); + assert_eq!(flash.read(LEN - 1, 1), 0xff); + program(&mut flash, 0x100, 0x1234_5678); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0x100, 1), 0x78); + assert_eq!(flash.read(0x101, 2), 0x3456); + assert_eq!(flash.read(0x100, 4), 0x1234_5678); + assert_eq!(flash.read(0x100, 8), 0xffff_ffff_1234_5678); + assert_eq!(flash.read(0x103, 4), 0xffff_ff12); + + command(&mut flash, 0x104, 0x10); + flash.write(0x104, 2, 0x70ff); // Command-looking bytes are still data. + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0x104, 4), 0xffff_70ff); + command(&mut flash, 0x106, 0x40); + flash.write(0x106, 1, 0x90); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0x104, 4), 0xff90_70ff); + } + + #[test] + fn programming_cannot_set_bits_and_errors_are_sticky_until_cleared() { + let mut flash = NorFlash::new(); + program(&mut flash, 0, 0); + program(&mut flash, 0, 0xffff_ffff); + assert_eq!(status(&mut flash), 0x0090_0090); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0, 4), 0); + program(&mut flash, 4, 0x1234); + assert_eq!(status(&mut flash), 0x0090_0090); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(4, 4), 0x1234); + assert_eq!(status(&mut flash), 0x0080_0080); + } + + #[test] + fn erase_requires_confirmation_and_changes_only_the_selected_block() { + let mut flash = NorFlash::new(); + program(&mut flash, 0, 0); + program(&mut flash, BLOCK_SIZE, 0); + command(&mut flash, 0, 0x20); + assert_eq!(flash.storage[0], 0); + command(&mut flash, 0, 0xff); // Cancel before confirmation. + assert_eq!(flash.read(0, 4), 0); + command(&mut flash, 0x100, 0x20); + command(&mut flash, 0, 0xd0); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(0, 4), 0xffff_ffff); + assert_eq!(flash.read(BLOCK_SIZE - 4, 4), 0xffff_ffff); + assert_eq!(flash.read(BLOCK_SIZE, 4), 0); + + command(&mut flash, BLOCK_SIZE, 0x20); + command(&mut flash, 0, 0xd0); // Wrong block must not erase anything. + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(BLOCK_SIZE, 4), 0); + } + + #[test] + fn ids_lock_unlock_and_cfi_describe_the_interleaved_bank() { + let mut flash = NorFlash::new(); + command(&mut flash, 0, 0x90); + assert_eq!(flash.read(0, 4), 0x0089_0089); + assert_eq!(flash.read(4, 4), 0x0018_0018); + assert_eq!(flash.read(0, 8), 0x0018_0018_0089_0089); + assert_eq!(flash.read(BLOCK_SIZE + 8, 4), 0); + + command(&mut flash, BLOCK_SIZE, 0x60); + command(&mut flash, BLOCK_SIZE, 0x01); + command(&mut flash, BLOCK_SIZE + 8, 0x90); + assert_eq!(flash.read(BLOCK_SIZE + 8, 4), 0x0001_0001); + program(&mut flash, BLOCK_SIZE, 0); + assert_eq!(status(&mut flash), 0x0092_0092); + command(&mut flash, 0, 0x50); + command(&mut flash, BLOCK_SIZE, 0x20); + command(&mut flash, BLOCK_SIZE, 0xd0); + assert_eq!(status(&mut flash), 0x00a2_00a2); + command(&mut flash, 0, 0x50); + command(&mut flash, BLOCK_SIZE, 0x60); + command(&mut flash, BLOCK_SIZE, 0xd0); + program(&mut flash, BLOCK_SIZE, 0); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(BLOCK_SIZE, 4), 0); + + command(&mut flash, 0x55 * 4, 0x98); + assert_eq!(flash.read(0x10 * 4, 4), 0x0051_0051); + assert_eq!(flash.read(0x11 * 4, 4), 0x0052_0052); + assert_eq!(flash.read(0x12 * 4, 4), 0x0059_0059); + assert_eq!(flash.read(0x27 * 4, 4), 0x0019_0019); + assert_eq!(flash.read(0x2a * 4, 4), 0x000b_000b); + assert_eq!(flash.read(0x2d * 4, 4), 0x00ff_00ff); + assert_eq!(flash.read(0x30 * 4, 4), 0x0002_0002); + assert_eq!(flash.read(0x52 * 4, 4), 0); + } + + #[test] + fn buffered_program_stages_data_until_base_address_confirmation() { + let mut flash = NorFlash::new(); + let target = BLOCK_SIZE + 128; + command(&mut flash, target, 0xe8); + assert_eq!(flash.read(target, 4), 0x0080_0080); + command(&mut flash, target, 1); // Two words. + flash.write(target, 4, 0x1234_5678); + flash.write(target + 4, 4, 0xd0ff_70e8); + assert_eq!(flash.storage[target as usize], 0xff); + command(&mut flash, 0, 0xd0); // EDK2 confirms at the bank base. + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(target, 8), 0xd0ff_70e8_1234_5678); + + command(&mut flash, target + 8, 0xe8); + command(&mut flash, target + 8, 0); // Count zero is one word. + flash.write(target + 8, 4, 0x1122_33ff); + command(&mut flash, 0, 0xd0); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(target + 8, 4), 0x1122_33ff); + + command(&mut flash, target, 0xe8); + command(&mut flash, target, 0); + flash.write(target, 4, 0xffff_ffff); + command(&mut flash, 0, 0xd0); + assert_eq!(status(&mut flash), 0x0090_0090); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(target, 4), 0x1234_5678); + } + + #[test] + fn full_edk2_buffer_and_last_bank_word_are_programmable() { + let mut flash = NorFlash::new(); + let target = LEN - 128; + command(&mut flash, target, 0xe8); + command(&mut flash, target, 31); + for index in 0..32 { + flash.write(target + index * 4, 4, 0xa500_0000 | index); + } + command(&mut flash, 0, 0xd0); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + for index in 0..32 { + assert_eq!(flash.read(target + index * 4, 4), 0xa500_0000 | index); + } + + let target = LEN - BUFFER_SIZE as u64; + command(&mut flash, target, 0xe8); + command(&mut flash, target, (BUFFER_WORDS - 1) as u16); + for index in 0..BUFFER_WORDS as u64 { + // Keep the earlier P30 data unchanged while testing the larger + // buffer capacity advertised by QEMU's CFI table. + let value = if index < BUFFER_WORDS as u64 - 32 { + 0x5a00_0000 | index + } else { + 0xa500_0000 | (index - (BUFFER_WORDS as u64 - 32)) + }; + flash.write(target + index * 4, 4, value); + } + command(&mut flash, 0, 0xd0); + assert_eq!(status(&mut flash), 0x0080_0080); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(target, 4), 0x5a00_0000); + assert_eq!(flash.read(LEN - 4, 4), 0xa500_001f); + } + + #[test] + fn invalid_buffer_sequences_discard_staged_data() { + let mut flash = NorFlash::new(); + command(&mut flash, 0, 0xe8); + command(&mut flash, 0, BUFFER_WORDS as u16); // One word too many. + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + + command(&mut flash, 0, 0xe8); + command(&mut flash, 0, 1); + flash.write(0, 4, 0); + flash.write(BUFFER_SIZE as u64, 4, 0); // Cross the write-buffer window. + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(0, 8), u64::MAX); + assert_eq!(flash.read(BUFFER_SIZE as u64, 4), 0xffff_ffff); + + command(&mut flash, 0, 0xe8); + command(&mut flash, 0, 0); + flash.write(0, 4, 0); + command(&mut flash, 0, 0x70); // Not a confirmation. + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(0, 4), 0xffff_ffff); + + command(&mut flash, 0, 0xe8); + command(&mut flash, 0, 0); + flash.write(0, 4, 0); + command(&mut flash, 0, 0xff); // Explicit cancellation. + assert_eq!(flash.read(0, 4), 0xffff_ffff); + } + + #[test] + fn invalid_commands_and_accesses_are_bounded_and_recoverable() { + let mut flash = NorFlash::new(); + assert_eq!(flash.read(LEN, 1), 0); + assert_eq!(flash.read(LEN - 3, 4), 0); + assert_eq!(flash.read(u64::MAX, 8), 0); + assert_eq!(flash.read(0, 0), 0); + assert_eq!(flash.read(0, 3), 0); + flash.write(LEN - 3, 4, 0); + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(LEN - 4, 4), 0xffff_ffff); + command(&mut flash, 0, 0xab); + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + program(&mut flash, LEN - 4, 0x1234_5678); + command(&mut flash, 0, 0xff); + assert_eq!(flash.read(LEN - 4, 4), 0x1234_5678); + + command(&mut flash, 0, 0x40); + flash.write(u64::MAX, 4, 0); + assert_eq!(status(&mut flash), 0x00b0_00b0); + command(&mut flash, 0, 0x50); + assert_eq!(flash.read(0, 4), 0xffff_ffff); + flash.write(0, 8, 0); + assert_eq!(status(&mut flash), 0x00b0_00b0); + } +} diff --git a/src/emulator/src/devices/pci.rs b/src/emulator/src/devices/pci.rs new file mode 100644 index 0000000..5c912e2 --- /dev/null +++ b/src/emulator/src/devices/pci.rs @@ -0,0 +1,427 @@ +//! PCI for the `virt` board: the root bus's configuration space (ECAM), and the memory +//! BARs of the functions on it. +//! +//! Only bus 0 is populated, one function per slot, and no bridges sit below it. Firmware +//! assigns the BARs itself, as it does under QEMU, so a function claims the addresses its +//! BARs hold once the command register enables memory decoding. + +use alloc::boxed::Box; +use alloc::vec::Vec; + +/// The ECAM window of QEMU `virt`'s high PCIe region: 256 buses of 4 KiB functions. +pub const ECAM_BASE: u64 = 0x40_1000_0000; +pub const ECAM_SIZE: u64 = 0x1000_0000; + +/// The host bridge QEMU's GPEX reports at 00:00.0. +const HOST_BRIDGE_VENDOR: u16 = 0x1b36; +const HOST_BRIDGE_DEVICE: u16 = 0x0008; +/// Base class 0x06, bridge; subclass 0x00, host bridge. +const HOST_BRIDGE_CLASS: u32 = 0x06_00_00; + +const CONFIG_LEN: usize = 256; +pub const BAR_COUNT: usize = 6; +const BAR_OFFSET: usize = 0x10; +const CAPABILITIES_POINTER: usize = 0x34; +/// Capabilities start after the 64-byte standard header. +const FIRST_CAPABILITY: usize = 0x40; +const STATUS: usize = 0x06; +const STATUS_CAPABILITIES: u8 = 0x10; +const COMMAND: usize = 0x04; +/// The command bits a driver may set: I/O, memory, bus master, special cycles, MWI, +/// VGA palette snoop, parity, SERR, fast back-to-back and INTx disable. +const COMMAND_WRITABLE: [u8; 2] = [0x47, 0x05]; +const COMMAND_MEMORY: u8 = 0x2; + +/// The 256-byte configuration space of a type-0 function. Writes honour a per-byte +/// writable mask, and a memory BAR keeps only the address bits its size allows, so the +/// standard sizing probe reads back the size. +pub struct ConfigSpace { + bytes: [u8; CONFIG_LEN], + writable: [u8; CONFIG_LEN], + /// Size of each memory BAR in bytes, zero when the BAR is not implemented. + bar_size: [u64; BAR_COUNT], + /// Offset of the last capability, whose `next` pointer is filled in by the next one. + last_capability: Option, + /// The first offset free for another capability, four-byte aligned. + capabilities_end: usize, +} + +impl ConfigSpace { + /// `class` is base class, subclass and programming interface, as `0xBBSSPP`. + pub fn new( + vendor: u16, + device: u16, + class: u32, + subsystem_vendor: u16, + subsystem: u16, + ) -> Self { + let mut config = Self { + bytes: [0; CONFIG_LEN], + writable: [0; CONFIG_LEN], + bar_size: [0; BAR_COUNT], + last_capability: None, + capabilities_end: FIRST_CAPABILITY, + }; + config.bytes[0x00..0x02].copy_from_slice(&vendor.to_le_bytes()); + config.bytes[0x02..0x04].copy_from_slice(&device.to_le_bytes()); + config.bytes[0x09] = class as u8; + config.bytes[0x0a] = (class >> 8) as u8; + config.bytes[0x0b] = (class >> 16) as u8; + config.bytes[0x2c..0x2e].copy_from_slice(&subsystem_vendor.to_le_bytes()); + config.bytes[0x2e..0x30].copy_from_slice(&subsystem.to_le_bytes()); + config.writable[COMMAND] = COMMAND_WRITABLE[0]; + config.writable[COMMAND + 1] = COMMAND_WRITABLE[1]; + config + } + + /// Declares memory BAR `index` of `size` bytes, a power of two of at least 16. + pub fn set_bar(&mut self, index: usize, size: u64) { + assert!(index < BAR_COUNT && size.is_power_of_two() && (16..=1 << 32).contains(&size)); + self.bar_size[index] = size; + let at = BAR_OFFSET + 4 * index; + self.writable[at..at + 4].fill(0xff); + } + + /// Appends a capability with identifier `id`. `body` is what follows the `next` + /// pointer, starting with the capability's own length byte. + pub fn add_capability(&mut self, id: u8, body: &[u8]) { + let at = self.capabilities_end; + let length = 2 + body.len(); + assert!( + at + length <= CONFIG_LEN, + "capability does not fit configuration space" + ); + self.bytes[at] = id; + self.bytes[at + 1] = 0; + self.bytes[at + 2..at + length].copy_from_slice(body); + match self.last_capability { + None => self.bytes[CAPABILITIES_POINTER] = at as u8, + Some(previous) => self.bytes[previous + 1] = at as u8, + } + self.bytes[STATUS] |= STATUS_CAPABILITIES; + self.last_capability = Some(at); + self.capabilities_end = (at + length + 3) & !3; + } + + /// The base and size of memory BAR `index`, once firmware has assigned it. + pub fn memory_bar(&self, index: usize) -> Option<(u64, u64)> { + let size = self.bar_size[index]; + if size == 0 { + return None; + } + let at = BAR_OFFSET + 4 * index; + let base = u32::from_le_bytes(self.bytes[at..at + 4].try_into().unwrap()); + Some((u64::from(base & !0xf), size)) + } + + /// Whether the command register lets the function decode its memory BARs. + pub fn memory_decoding(&self) -> bool { + self.bytes[COMMAND] & COMMAND_MEMORY != 0 + } + + /// Reads `size` bytes, little-endian. Bytes past the header read as zero. + pub fn read(&self, offset: usize, size: usize) -> u64 { + (0..size).fold(0, |value, i| { + let byte = self.bytes.get(offset + i).copied().unwrap_or(0); + value | u64::from(byte) << (8 * i) + }) + } + + /// Writes `size` bytes, little-endian, keeping the bytes the writable mask protects. + pub fn write(&mut self, offset: usize, size: usize, value: u64) { + for i in 0..size { + let at = offset + i; + if at >= CONFIG_LEN { + break; + } + let mask = self.writable[at]; + let byte = (value >> (8 * i)) as u8; + self.bytes[at] = (self.bytes[at] & !mask) | (byte & mask); + } + for index in 0..BAR_COUNT { + let size = self.bar_size[index]; + if size == 0 { + continue; + } + let at = BAR_OFFSET + 4 * index; + let written = u32::from_le_bytes(self.bytes[at..at + 4].try_into().unwrap()); + let kept = written & !((size - 1) as u32) & !0xf; + self.bytes[at..at + 4].copy_from_slice(&kept.to_le_bytes()); + } + } +} + +/// A function on the root bus. Its BARs are reached through the bus, which decides which +/// function a bus address belongs to. +pub trait PciFunction { + fn config(&self) -> &ConfigSpace; + fn config_mut(&mut self) -> &mut ConfigSpace; + /// Reads `size` bytes at `offset` into memory BAR `bar`. + fn bar_read(&mut self, bar: usize, offset: u64, size: u8) -> u64 { + let _ = (bar, offset, size); + 0 + } + /// Writes `size` bytes at `offset` into memory BAR `bar`. + fn bar_write(&mut self, bar: usize, offset: u64, size: u8, value: u64) { + let _ = (bar, offset, size, value); + } +} + +/// The root bus: functions attached to slots, served through ECAM and their BARs. +#[derive(Default)] +pub struct PciHost { + slots: Vec<(u8, Box)>, +} + +impl PciHost { + /// A bus with the GPEX host bridge at 00:00.0 and nothing else. + pub fn new() -> Self { + let mut host = Self::default(); + host.attach(0, Box::new(HostBridge::new())); + host + } + + /// Attaches `function` at `slot` on bus 0. + pub fn attach(&mut self, slot: u8, function: Box) { + assert!(slot < 32, "slot {slot} is not on bus 0"); + assert!( + self.slots.iter().all(|(taken, _)| *taken != slot), + "slot {slot} is already occupied" + ); + self.slots.push((slot, function)); + } + + /// Reads `size` bytes at a physical address the bus claims, or `None` for one it does not. + pub fn read(&mut self, address: u64, size: u8) -> Option { + if let Some(ecam) = ecam_offset(address) { + return Some(self.config_read(ecam, size)); + } + let (index, bar, offset) = self.claim(address)?; + Some(self.slots[index].1.bar_read(bar, offset, size)) + } + + /// Writes `size` bytes at a physical address the bus claims. `false` declines one it does not. + pub fn write(&mut self, address: u64, size: u8, value: u64) -> bool { + if let Some(ecam) = ecam_offset(address) { + self.config_write(ecam, size, value); + return true; + } + let Some((index, bar, offset)) = self.claim(address) else { + return false; + }; + self.slots[index].1.bar_write(bar, offset, size, value); + true + } + + fn config_read(&self, ecam: u64, size: u8) -> u64 { + let all_ones = size_mask(size); + match self.function(ecam) { + Some((function, register)) => function.config().read(register, size as usize), + None => all_ones, + } + } + + fn config_write(&mut self, ecam: u64, size: u8, value: u64) { + let Some(slot) = self.function_index(ecam) else { + return; + }; + let register = (ecam & 0xfff) as usize; + self.slots[slot] + .1 + .config_mut() + .write(register, size as usize, value); + } + + /// The function an ECAM offset names, and the register within it. Only bus 0 and + /// function 0 are populated. + fn function(&self, ecam: u64) -> Option<(&dyn PciFunction, usize)> { + let index = self.function_index(ecam)?; + Some((self.slots[index].1.as_ref(), (ecam & 0xfff) as usize)) + } + + fn function_index(&self, ecam: u64) -> Option { + let bus = ecam >> 20; + let slot = ((ecam >> 15) & 0x1f) as u8; + let function = (ecam >> 12) & 0x7; + if bus != 0 || function != 0 { + return None; + } + self.slots.iter().position(|(taken, _)| *taken == slot) + } + + /// The function, BAR and offset a bus address falls in. Only functions that enable + /// memory decoding answer. + fn claim(&self, address: u64) -> Option<(usize, usize, u64)> { + for (index, (_, function)) in self.slots.iter().enumerate() { + let config = function.config(); + if !config.memory_decoding() { + continue; + } + for bar in 0..BAR_COUNT { + if let Some((base, size)) = config.memory_bar(bar) + && (base..base + size).contains(&address) + { + return Some((index, bar, address - base)); + } + } + } + None + } +} + +fn ecam_offset(address: u64) -> Option { + (ECAM_BASE..ECAM_BASE + ECAM_SIZE) + .contains(&address) + .then(|| address - ECAM_BASE) +} + +fn size_mask(size: u8) -> u64 { + if size >= 8 { + u64::MAX + } else { + (1u64 << (8 * size)) - 1 + } +} + +/// The GPEX host bridge at 00:00.0. It has no BARs; its configuration space is what +/// enumeration reads first. +pub struct HostBridge { + config: ConfigSpace, +} + +impl HostBridge { + fn new() -> Self { + Self { + config: ConfigSpace::new( + HOST_BRIDGE_VENDOR, + HOST_BRIDGE_DEVICE, + HOST_BRIDGE_CLASS, + 0, + 0, + ), + } + } +} + +impl PciFunction for HostBridge { + fn config(&self) -> &ConfigSpace { + &self.config + } + + fn config_mut(&mut self) -> &mut ConfigSpace { + &mut self.config + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloc::rc::Rc; + use core::cell::RefCell; + + type Writes = Rc>>; + + /// A function with one 16 KiB BAR. Its BAR reads return the offset, and its writes are + /// recorded where the test can see them. + struct Probe { + config: ConfigSpace, + writes: Writes, + } + + impl Probe { + fn new(writes: Writes) -> Self { + let mut config = ConfigSpace::new(0x1af4, 0x1042, 0x018000, 0x1af4, 2); + config.set_bar(0, 0x4000); + // A capability, so the status register's capability-list bit is set as on virtio. + let mut body = [0u8; 14]; + body[0] = 16; + config.add_capability(0x09, &body); + Self { config, writes } + } + } + + impl PciFunction for Probe { + fn config(&self) -> &ConfigSpace { + &self.config + } + fn config_mut(&mut self) -> &mut ConfigSpace { + &mut self.config + } + fn bar_read(&mut self, bar: usize, offset: u64, _size: u8) -> u64 { + if bar == 0 { offset } else { 0 } + } + fn bar_write(&mut self, bar: usize, offset: u64, size: u8, value: u64) { + self.writes.borrow_mut().push((bar, offset, size, value)); + } + } + + fn ecam(bus: u64, slot: u64, function: u64, register: u64) -> u64 { + ECAM_BASE | bus << 20 | slot << 15 | function << 12 | register + } + + fn probe_host() -> (PciHost, Writes) { + let writes = Writes::default(); + let mut host = PciHost::new(); + host.attach(1, Box::new(Probe::new(writes.clone()))); + (host, writes) + } + + #[test] + fn host_bridge_identifies_itself_and_empty_slots_read_as_all_ones() { + let mut host = PciHost::new(); + assert_eq!( + host.read(ecam(0, 0, 0, 0), 4), + Some(u64::from(HOST_BRIDGE_VENDOR) | u64::from(HOST_BRIDGE_DEVICE) << 16) + ); + assert_eq!(host.read(ecam(0, 5, 0, 0), 4), Some(0xffff_ffff)); + assert_eq!(host.read(ecam(0, 5, 0, 0), 2), Some(0xffff)); + assert_eq!(host.read(ecam(1, 0, 0, 0), 4), Some(0xffff_ffff)); + assert_eq!(host.read(ecam(0, 0, 3, 0), 4), Some(0xffff_ffff)); + } + + #[test] + fn bar_sizing_probe_reads_back_the_size_mask_and_restores_the_address() { + let (mut host, _) = probe_host(); + let bar = ecam(0, 1, 0, 0x10); + host.write(bar, 4, 0x1000_0000); + assert_eq!(host.read(bar, 4), Some(0x1000_0000)); + // All ones reads back as the writable address bits, which name 16 KiB. + host.write(bar, 4, 0xffff_ffff); + assert_eq!(host.read(bar, 4), Some(0xffff_c000)); + host.write(bar, 4, 0x1000_0000); + assert_eq!(host.read(bar, 4), Some(0x1000_0000)); + } + + #[test] + fn a_bar_answers_only_while_memory_decoding_is_enabled() { + let (mut host, _) = probe_host(); + host.write(ecam(0, 1, 0, 0x10), 4, 0x1000_0000); + assert_eq!(host.read(0x1000_0004, 4), None); + host.write(ecam(0, 1, 0, 0x04), 2, 0x0002); + assert_eq!(host.read(0x1000_0004, 4), Some(4)); + assert_eq!( + host.read(0x1000_4000, 4), + None, + "one past the BAR is unclaimed" + ); + assert!(!host.write(0x1000_4000, 4, 0)); + } + + #[test] + fn a_write_into_a_bar_reaches_the_function_at_its_offset() { + let (mut host, writes) = probe_host(); + host.write(ecam(0, 1, 0, 0x10), 4, 0x1000_0000); + host.write(ecam(0, 1, 0, 0x04), 2, 0x0002); + assert!(host.write(0x1000_0040, 2, 0xabcd)); + assert_eq!(writes.borrow().as_slice(), &[(0, 0x40, 2, 0xabcd)]); + } + + #[test] + fn the_command_register_keeps_only_writable_bits_and_status_lists_capabilities() { + let (mut host, _) = probe_host(); + // Status is read only: its capability-list bit (bit 4) survives the write. + host.write(ecam(0, 1, 0, 0x04), 4, 0xffff_ffff); + assert_eq!(host.read(ecam(0, 1, 0, 0x04), 2), Some(0x0547)); + assert_eq!(host.read(ecam(0, 1, 0, 0x06), 2), Some(0x0010)); + } +} diff --git a/src/emulator/src/devices/pl031.rs b/src/emulator/src/devices/pl031.rs new file mode 100644 index 0000000..3f48942 --- /dev/null +++ b/src/emulator/src/devices/pl031.rs @@ -0,0 +1,286 @@ +//! QEMU virt's PL031 real-time clock, driven by the host monotonic clock. +//! +//! Like QEMU's PL031, the counter is permanently enabled. Alarm status is +//! latched when the device is accessed; no interrupt line is connected. + +use core::time::Duration; +use std::time::Instant; + +use super::bus::Device; + +pub const BASE: u64 = 0x0901_0000; +pub const LEN: u64 = 0x1000; + +const DR: u64 = 0x00; +const MR: u64 = 0x04; +const LR: u64 = 0x08; +const CR: u64 = 0x0c; +const IMSC: u64 = 0x10; +const RIS: u64 = 0x14; +const MIS: u64 = 0x18; +const ICR: u64 = 0x1c; +const PERIPH_ID0: u64 = 0xfe0; +const IDENTIFICATION: [u32; 8] = [0x31, 0x10, 0x14, 0x00, 0x0d, 0xf0, 0x05, 0xb1]; + +/// A wrapping 32-bit seconds counter with PL031 alarm and identification registers. +pub struct Pl031 { + origin: Instant, + tick_offset: u32, + load: u32, + match_value: u32, + interrupt_mask: u32, + raw_interrupt: u32, + alarm_deadline: Option, +} + +impl Pl031 { + /// Start at `epoch_seconds`; subsequent time is elapsed host monotonic time. + pub fn new(epoch_seconds: u32) -> Self { + Self::new_at(epoch_seconds, Instant::now()) + } + + fn new_at(epoch_seconds: u32, now: Instant) -> Self { + Self { + origin: now, + tick_offset: epoch_seconds, + load: 0, + match_value: 0, + interrupt_mask: 0, + raw_interrupt: 0, + alarm_deadline: None, + } + } + + fn count(&self, elapsed: Duration) -> u32 { + self.tick_offset.wrapping_add(elapsed.as_secs() as u32) + } + + fn refresh_alarm(&mut self, elapsed: Duration) { + if self + .alarm_deadline + .is_some_and(|deadline| elapsed >= deadline) + { + self.raw_interrupt = 1; + self.alarm_deadline = None; + } + } + + fn set_alarm(&mut self, elapsed: Duration) { + // Unsigned subtraction schedules a past match after the counter wraps. + // QEMU raises an already matching alarm immediately, otherwise schedules + // it this many seconds after programming MR or LR. + let ticks = self.match_value.wrapping_sub(self.count(elapsed)); + if ticks == 0 { + self.raw_interrupt = 1; + self.alarm_deadline = None; + } else { + self.alarm_deadline = Some(elapsed + Duration::from_secs(u64::from(ticks))); + } + } + + fn access(offset: u64, size: u8) -> Option<(u64, u32, u32)> { + // The APB register width is 32 bits. Byte and halfword accesses select + // little-endian lanes, including the byte-sized PrimeCell IDs. + if !matches!(size, 1 | 2 | 4) || offset >= LEN || (offset & 3) + u64::from(size) > 4 { + return None; + } + let shift = ((offset & 3) * 8) as u32; + let mask = u32::MAX >> (32 - u32::from(size) * 8); + Some((offset & !3, shift, mask)) + } + + fn read_at(&mut self, offset: u64, size: u8, now: Instant) -> u64 { + let Some((register, shift, mask)) = Self::access(offset, size) else { + return 0; + }; + let elapsed = now.duration_since(self.origin); + self.refresh_alarm(elapsed); + let value = match register { + DR => self.count(elapsed), + MR => self.match_value, + LR => self.load, + CR => 1, // QEMU's RTC is permanently enabled. + IMSC => self.interrupt_mask, + RIS => self.raw_interrupt, + MIS => self.raw_interrupt & self.interrupt_mask, + PERIPH_ID0..=0xffc => IDENTIFICATION[((register - PERIPH_ID0) / 4) as usize], + _ => 0, // Includes the write-only interrupt clear register. + }; + u64::from((value >> shift) & mask) + } + + fn write_at(&mut self, offset: u64, size: u8, value: u64, now: Instant) { + let Some((register, shift, mask)) = Self::access(offset, size) else { + return; + }; + let elapsed = now.duration_since(self.origin); + self.refresh_alarm(elapsed); + let bits = ((value as u32) & mask) << shift; + let lane_mask = mask << shift; + match register { + MR => { + self.match_value = (self.match_value & !lane_mask) | bits; + self.set_alarm(elapsed); + } + LR => { + self.load = (self.load & !lane_mask) | bits; + // Keep the oscillator's subsecond phase when setting the time. + self.tick_offset = self + .tick_offset + .wrapping_add(self.load.wrapping_sub(self.count(elapsed))); + self.set_alarm(elapsed); + } + IMSC => self.interrupt_mask = ((self.interrupt_mask & !lane_mask) | bits) & 1, + ICR => self.raw_interrupt &= !bits, + // CR writes are ignored, as in QEMU; all remaining registers are + // read-only or reserved. + _ => {} + } + } +} + +impl Device for Pl031 { + fn read(&mut self, offset: u64, size: u8) -> u64 { + self.read_at(offset, size, Instant::now()) + } + + fn write(&mut self, offset: u64, size: u8, value: u64) { + self.write_at(offset, size, value, Instant::now()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counter_advances_in_whole_seconds_and_wraps() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(u32::MAX - 1, start); + assert_eq!(rtc.read_at(DR, 4, start), u64::from(u32::MAX - 1)); + assert_eq!( + rtc.read_at(DR, 4, start + Duration::from_millis(999)), + u64::from(u32::MAX - 1) + ); + assert_eq!( + rtc.read_at(DR, 4, start + Duration::from_secs(1)), + u64::from(u32::MAX) + ); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(2)), 0); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(3)), 1); + } + + #[test] + fn loading_time_preserves_fractional_phase_and_load_readback() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(100, start); + rtc.write_at(LR, 4, 500, start + Duration::from_millis(1750)); + assert_eq!(rtc.read_at(LR, 4, start + Duration::from_millis(1750)), 500); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_millis(1999)), 500); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(2)), 501); + assert_eq!(rtc.read_at(LR, 4, start + Duration::from_secs(2)), 500); + rtc.write_at(LR, 4, 20, start + Duration::from_secs(3)); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(3)), 20); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(4)), 21); + } + + #[test] + fn control_is_permanently_enabled_like_qemu() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(100, start); + assert_eq!(rtc.read_at(CR, 4, start), 1); + rtc.write_at(CR, 4, 0, start); + assert_eq!(rtc.read_at(CR, 4, start), 1); + assert_eq!(rtc.read_at(DR, 4, start + Duration::from_secs(2)), 102); + rtc.write_at(CR, 4, u64::MAX, start + Duration::from_secs(2)); + assert_eq!(rtc.read_at(CR, 4, start + Duration::from_secs(2)), 1); + } + + #[test] + fn match_latches_raw_status_independently_of_the_mask() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(100, start); + rtc.write_at(MR, 4, 102, start); + assert_eq!(rtc.read_at(MR, 4, start), 102); + assert_eq!(rtc.read_at(RIS, 4, start + Duration::from_secs(1)), 0); + let after = start + Duration::from_secs(5); + // A missed matching second still latches the interrupt. + assert_eq!(rtc.read_at(RIS, 4, after), 1); + assert_eq!(rtc.read_at(MIS, 4, after), 0); + rtc.write_at(IMSC, 4, 0xffff, after); + assert_eq!(rtc.read_at(IMSC, 4, after), 1); + assert_eq!(rtc.read_at(MIS, 4, after), 1); + rtc.write_at(IMSC, 4, 0, after); + assert_eq!(rtc.read_at(MIS, 4, after), 0); + assert_eq!(rtc.read_at(RIS, 4, after), 1); + rtc.write_at(ICR, 4, 2, after); + assert_eq!(rtc.read_at(RIS, 4, after), 1); + rtc.write_at(ICR, 4, 1, after); + assert_eq!(rtc.read_at(RIS, 4, after), 0); + assert_eq!(rtc.read_at(ICR, 4, after), 0); + } + + #[test] + fn match_scheduling_wraps_and_load_reschedules_the_alarm() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(u32::MAX - 1, start); + rtc.write_at(MR, 4, 1, start); + assert_eq!(rtc.read_at(RIS, 4, start + Duration::from_secs(2)), 0); + assert_eq!(rtc.read_at(RIS, 4, start + Duration::from_secs(3)), 1); + let now = start + Duration::from_secs(3); + rtc.write_at(ICR, 4, 1, now); + assert_eq!(rtc.read_at(RIS, 4, now), 0); + rtc.write_at(MR, 4, 1, now); // Equal to the current count. + assert_eq!(rtc.read_at(RIS, 4, now), 1); + rtc.write_at(ICR, 4, 1, now); + rtc.write_at(MR, 4, 10, now); + rtc.write_at(LR, 4, 9, now); + assert_eq!(rtc.read_at(RIS, 4, now), 0); + assert_eq!(rtc.read_at(RIS, 4, now + Duration::from_secs(1)), 1); + rtc.write_at(ICR, 4, 1, now + Duration::from_secs(1)); + rtc.write_at(LR, 4, 10, now + Duration::from_secs(1)); + assert_eq!(rtc.read_at(RIS, 4, now + Duration::from_secs(1)), 1); + } + + #[test] + fn primecell_ids_and_little_endian_register_lanes_are_readable() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(0x1234_5678, start); + for (index, expected) in IDENTIFICATION.into_iter().enumerate() { + let offset = PERIPH_ID0 + index as u64 * 4; + assert_eq!(rtc.read_at(offset, 1, start), u64::from(expected)); + assert_eq!(rtc.read_at(offset, 4, start), u64::from(expected)); + assert_eq!(rtc.read_at(offset + 1, 1, start), 0); + } + assert_eq!(rtc.read_at(DR, 1, start), 0x78); + assert_eq!(rtc.read_at(DR + 1, 1, start), 0x56); + assert_eq!(rtc.read_at(DR + 2, 2, start), 0x1234); + rtc.write_at(MR, 4, 0x1122_3344, start); + rtc.write_at(MR + 1, 1, 0xaa, start); + assert_eq!(rtc.read_at(MR, 4, start), 0x1122_aa44); + rtc.write_at(LR, 4, 0x0102_0304, start); + rtc.write_at(LR + 2, 2, 0xaabb, start); + assert_eq!(rtc.read_at(LR, 4, start), 0xaabb_0304); + assert_eq!(rtc.read_at(DR, 4, start), 0xaabb_0304); + } + + #[test] + fn read_only_reserved_and_invalid_accesses_do_not_change_registers() { + let start = Instant::now(); + let mut rtc = Pl031::new_at(100, start); + for offset in [DR, RIS, MIS, PERIPH_ID0, 0x20, LEN, u64::MAX] { + rtc.write_at(offset, 4, u64::MAX, start); + } + assert_eq!(rtc.read_at(DR, 4, start), 100); + assert_eq!(rtc.read_at(RIS, 4, start), 0); + assert_eq!(rtc.read_at(MIS, 4, start), 0); + assert_eq!(rtc.read_at(PERIPH_ID0, 1, start), 0x31); + for (offset, size) in [(MR, 0), (MR, 3), (MR, 8), (MR + 3, 2), (LEN, 4)] { + rtc.write_at(offset, size, u64::MAX, start); + assert_eq!(rtc.read_at(offset, size, start), 0); + } + assert_eq!(rtc.read_at(MR, 4, start), 0); + assert_eq!(rtc.read_at(0x20, 4, start), 0); + assert_eq!(rtc.read_at(LEN, 4, start), 0); + } +} diff --git a/src/emulator/src/devices/pmgr.rs b/src/emulator/src/devices/pmgr.rs new file mode 100644 index 0000000..a2cba60 --- /dev/null +++ b/src/emulator/src/devices/pmgr.rs @@ -0,0 +1,93 @@ +//! Apple's power manager (PMGR), as the M1 Linux power-domain driver drives it. +//! +//! Each power domain has one 32-bit power-state register (`apple-pmgr-pwrstate`): +//! `PS_TARGET` in bits 3:0 is the state the driver asks for, and `PS_ACTUAL` in bits +//! 7:4 is the state the hardware is in. The driver writes the target and polls until +//! the actual state matches. This model acknowledges a write at once, so the actual +//! state is the target, and every register starts in the active state iBoot leaves +//! the always-on domains in. Other bits are stored as written. +use alloc::vec; +use alloc::vec::Vec; + +/// `PS_ACTIVE`: the target and the actual state both name the active state. +const ACTIVE: u32 = 0xff; +const TARGET: u32 = 0x0f; +const ACTUAL: u32 = 0xf0; + +/// The power-state registers of one PMGR block of `len` bytes. +pub struct Pmgr { + words: Vec, +} + +impl Pmgr { + /// A block of `len` bytes (a multiple of four) with every power domain active. + pub fn new(len: u64) -> Self { + Self { + words: vec![ACTIVE; (len / 4) as usize], + } + } + + /// Read `size` bytes at `offset` (relative to the block base). + pub fn read(&self, offset: u64, size: u8) -> u64 { + (0..u64::from(size)).fold(0, |value, i| { + let at = offset + i; + let word = self.words[(at / 4) as usize]; + value | u64::from((word >> (8 * (at % 4))) as u8) << (8 * i) + }) + } + + /// Write the low `size` bytes of `value` at `offset`. Each 32-bit register the write + /// touches takes its target power state at once. + pub fn write(&mut self, offset: u64, size: u8, value: u64) { + for i in 0..u64::from(size) { + let at = offset + i; + let shift = 8 * (at % 4); + let byte = ((value >> (8 * i)) & 0xff) as u32; + let word = &mut self.words[(at / 4) as usize]; + *word = (*word & !(0xff << shift)) | byte << shift; + } + let first = (offset / 4) as usize; + let last = ((offset + u64::from(size) - 1) / 4) as usize; + for word in &mut self.words[first..=last] { + *word = (*word & !ACTUAL) | (*word & TARGET) << 4; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PS: u64 = 0x100; + const LEN: u64 = 0x4000; + + #[test] + fn every_power_domain_starts_active() { + let pmgr = Pmgr::new(LEN); + assert_eq!(pmgr.read(PS, 4) & 0xff, 0xff); + } + + #[test] + fn a_target_is_acknowledged_as_the_actual_state_at_once() { + let mut pmgr = Pmgr::new(LEN); + pmgr.write(PS, 4, 0x0); + let state = pmgr.read(PS, 4); + assert_eq!(state & 0x0f, 0, "target is what was written"); + assert_eq!((state >> 4) & 0x0f, 0, "actual follows the target"); + pmgr.write(PS, 4, 0x0f | 1 << 31); + assert_eq!( + pmgr.read(PS, 4) & 0xff, + 0xff, + "active again, reset bit stored" + ); + } + + #[test] + fn narrow_writes_change_only_their_bytes_and_flags_are_stored() { + let mut pmgr = Pmgr::new(LEN); + // WAS_PWRGATED (bit 8) lives in the second byte: a byte write leaves the rest. + pmgr.write(PS + 1, 1, 0x1); + assert_eq!(pmgr.read(PS + 1, 1), 0x1); + assert_eq!(pmgr.read(PS, 1), 0xff); + } +} diff --git a/src/emulator/src/devices/s5l_uart.rs b/src/emulator/src/devices/s5l_uart.rs new file mode 100644 index 0000000..6566e3b --- /dev/null +++ b/src/emulator/src/devices/s5l_uart.rs @@ -0,0 +1,358 @@ +//! Apple S5L UART (`apple,s5l-uart`), the console port of M1-class SoCs. +//! +//! The Apple variant differs from the classic Samsung register map in one way +//! that matters here: it has no `UINTP`/`UINTM` interrupt registers. Its +//! interrupt causes are the flag bits of `UTRSTAT` itself. A cause is cleared +//! by writing a one to its bit, and a `UCON` enable bit decides whether a +//! pending cause reaches the line. `apple_serial_handle_irq` in the driver +//! works this way and never touches `UINTP`. +//! +//! Transmit completes at once. Every byte written to `UTXH` reaches the sink, +//! and the transmit FIFO never reports full. The one transmit cause is +//! `TXTHRESH` (`UTRSTAT` bit 5). Real hardware latches it when the transmit +//! FIFO drops below its threshold, and in this model every transmit does that, +//! so the cause is raised after each byte. The line rises only while +//! `UCON.TXTHRESH_ENA` is set, which is when the driver wants a TX interrupt. +//! +//! The driver's TX path never waits on that interrupt. `enable_tx_pio` sets the +//! enable bit and calls the TX routine itself, because the Apple UART only +//! raises edge-triggered TX interrupts (see the comment there). That routine +//! drains the queue synchronously, since the FIFO never fills, and then clears +//! the enable bit through `stop_tx`. If an interrupt does arrive while work is +//! left, the handler clears `TXTHRESH` first and then writes more bytes, which +//! raises the cause again, so each entry drains some of the queue. +//! +//! Receive has no input in this model: the receive FIFO is empty, `RXDR` stays +//! clear, and no receive cause is ever raised. + +use alloc::boxed::Box; +use alloc::collections::VecDeque; + +/// Mapping length of the UART register window. +pub const LEN: u64 = 0x1000; + +const ULCON: u64 = 0x00; +const UCON: u64 = 0x04; +const UFCON: u64 = 0x08; +const UMCON: u64 = 0x0c; +const UTRSTAT: u64 = 0x10; +const UFSTAT: u64 = 0x18; +const UTXH: u64 = 0x20; +const URXH: u64 = 0x24; +const UBRDIV: u64 = 0x28; +const UFRACVAL: u64 = 0x2c; + +/// `UCON` bit that enables the transmit interrupt line, `APPLE_S5L_UCON_TXTHRESH_ENA`. +const UCON_TXTHRESH_ENA: u32 = 1 << 13; + +/// `UTRSTAT` bit for the transmit-threshold cause, `APPLE_S5L_UTRSTAT_TXTHRESH`. +const UTRSTAT_TXTHRESH: u32 = 1 << 5; +/// Transmitter empty and transmit FIFO empty. Always set: transmit is instant. +const UTRSTAT_TXE: u32 = 1 << 2; +const UTRSTAT_TXFE: u32 = 1 << 1; +/// Receive data ready, set only while the receive FIFO holds data. +const UTRSTAT_RXDR: u32 = 1 << 0; + +/// `UFCON` bits that reset a FIFO. The hardware clears them itself, so they +/// never read back as set. +const UFCON_RESETRX: u32 = 1 << 1; +const UFCON_RESETTX: u32 = 1 << 2; + +/// `UFSTAT` receive count field and its full flag. +const UFSTAT_RXMASK: u32 = 0xf; +const UFSTAT_RXFULL: u32 = 1 << 8; +/// Receive FIFO depth reported in `UFSTAT`, `fifosize` for this port. +const FIFO_DEPTH: usize = 16; + +/// An Apple S5L UART. +/// +/// The sink takes each transmitted byte in order. The interrupt line is read +/// through [`S5lUart::irq_asserted`]. +pub struct S5lUart { + sink: Box, + ulcon: u32, + ucon: u32, + ufcon: u32, + umcon: u32, + ubrdiv: u32, + ufracval: u32, + /// Pending interrupt causes, in `UTRSTAT` bit positions. + pending: u32, + rx: VecDeque, +} + +impl S5lUart { + /// A UART writing transmitted bytes to `sink`. + pub fn new(sink: impl FnMut(u8) + Send + 'static) -> Self { + Self { + sink: Box::new(sink), + ulcon: 0, + ucon: 0, + ufcon: 0, + umcon: 0, + ubrdiv: 0, + ufracval: 0, + pending: 0, + rx: VecDeque::new(), + } + } + + /// Read a register. `offset` is relative to the device base; `size` is in + /// bytes. Unknown offsets read as zero. + pub fn read(&mut self, offset: u64, size: u8) -> u64 { + let value = match offset { + ULCON => self.ulcon, + UCON => self.ucon, + UFCON => self.ufcon, + UMCON => self.umcon, + UTRSTAT => self.utrstat(), + UFSTAT => self.ufstat(), + URXH => self.rx.pop_front().map_or(0, u32::from), + UBRDIV => self.ubrdiv, + UFRACVAL => self.ufracval, + _ => 0, + }; + u64::from(value) & lane_mask(size) + } + + /// Write a register. Only the low `size` bytes of `value` are used. + /// Unknown offsets and read-only registers ignore the write. + pub fn write(&mut self, offset: u64, size: u8, value: u64) { + let value = (value & lane_mask(size)) as u32; + match offset { + ULCON => self.ulcon = value, + UCON => self.ucon = value, + UFCON => { + if value & UFCON_RESETRX != 0 { + self.rx.clear(); + } + self.ufcon = value & !(UFCON_RESETRX | UFCON_RESETTX); + } + UMCON => self.umcon = value, + // Write-one-to-clear. Bits that are not causes have nothing pending + // to clear, so they fall through harmlessly. + UTRSTAT => self.pending &= !value, + UTXH => self.transmit(value as u8), + UBRDIV => self.ubrdiv = value, + UFRACVAL => self.ufracval = value, + _ => {} + } + } + + /// Level of the interrupt line to the AIC: asserted while the transmit + /// cause is pending and its `UCON` enable bit is set. + pub fn irq_asserted(&self) -> bool { + self.pending & UTRSTAT_TXTHRESH != 0 && self.ucon & UCON_TXTHRESH_ENA != 0 + } + + fn transmit(&mut self, byte: u8) { + (self.sink)(byte); + self.pending |= UTRSTAT_TXTHRESH; + } + + fn utrstat(&self) -> u32 { + let rxdr = if self.rx.is_empty() { 0 } else { UTRSTAT_RXDR }; + self.pending | UTRSTAT_TXE | UTRSTAT_TXFE | rxdr + } + + /// The transmit FIFO never holds anything, so only the receive count shows. + fn ufstat(&self) -> u32 { + let len = self.rx.len(); + let full = if len >= FIFO_DEPTH { UFSTAT_RXFULL } else { 0 }; + full | (len as u32 & UFSTAT_RXMASK) + } + + /// Feed one byte into the receive FIFO, as input from a host would. + #[cfg(test)] + fn feed_rx(&mut self, byte: u8) { + self.rx.push_back(byte); + } +} + +/// Mask for the low `size` bytes of a register access. +fn lane_mask(size: u8) -> u64 { + match size { + 1 => 0xff, + 2 => 0xffff, + 4 => 0xffff_ffff, + _ => u64::MAX, + } +} + +#[cfg(test)] +mod tests { + use super::{ + S5lUart, UBRDIV, UCON, UCON_TXTHRESH_ENA, UFCON, UFRACVAL, UFSTAT, ULCON, UMCON, URXH, + UTRSTAT, UTRSTAT_TXTHRESH, UTXH, + }; + use alloc::sync::Arc; + use core::sync::atomic::{AtomicU8, AtomicUsize, Ordering}; + + const CAPACITY: usize = 512; + const ZERO: AtomicU8 = AtomicU8::new(0); + + /// Bytes the sink received, shared with the test through atomics so the + /// sink stays `Send` without `std`. + struct Capture { + len: AtomicUsize, + bytes: [AtomicU8; CAPACITY], + } + + impl Capture { + fn new() -> Arc { + Arc::new(Self { + len: AtomicUsize::new(0), + bytes: [ZERO; CAPACITY], + }) + } + + fn push(&self, byte: u8) { + let index = self.len.fetch_add(1, Ordering::SeqCst); + self.bytes[index].store(byte, Ordering::SeqCst); + } + + fn collected(&self) -> alloc::vec::Vec { + (0..self.len.load(Ordering::SeqCst)) + .map(|i| self.bytes[i].load(Ordering::SeqCst)) + .collect() + } + } + + fn uart_with_capture() -> (S5lUart, Arc) { + let capture = Capture::new(); + let sink = Arc::clone(&capture); + (S5lUart::new(move |byte| sink.push(byte)), capture) + } + + /// The `UFCON` value the driver's startup writes: FIFO mode, RX trigger 8, + /// and the RX reset bit it ORs in (plus TX reset when not a console). + const UFCON_STARTUP: u32 = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 4); + + #[test] + fn transmitted_bytes_reach_sink_in_order() { + let (mut uart, capture) = uart_with_capture(); + for &byte in b"volt\n" { + uart.write(UTXH, 4, u64::from(byte)); + } + // Only the low byte of the register is the character. + uart.write(UTXH, 4, 0x1_0000_0041); + uart.write(UTXH, 1, 0x142); + assert_eq!(capture.collected(), b"volt\nAB"); + } + + #[test] + fn transmit_fifo_never_reports_full_or_nonempty() { + let (mut uart, _capture) = uart_with_capture(); + for i in 0..200u64 { + uart.write(UTXH, 4, i); + let ufstat = uart.read(UFSTAT, 4); + assert_eq!(ufstat & (1 << 9), 0, "TXFULL set after byte {i}"); + assert_eq!(ufstat & (0xf << 4), 0, "TX count nonzero after byte {i}"); + } + } + + #[test] + fn early_console_sees_room_and_empty_transmitter() { + // `samsung_early_putc` polls TXFULL when UFCON has FIFO mode set, and + // TXFE in UTRSTAT otherwise. The driver's startup sets FIFO mode. + let (mut uart, _capture) = uart_with_capture(); + uart.write(UFCON, 4, u64::from(UFCON_STARTUP)); + uart.write(UTXH, 4, u64::from(b'x')); + assert_eq!(uart.read(UFSTAT, 4) & (1 << 9), 0); + assert_ne!(uart.read(UTRSTAT, 4) & (1 << 1), 0); + // Without FIFO mode the early console polls TXFE only; it must also pass. + uart.write(UFCON, 4, 0); + assert_ne!(uart.read(UTRSTAT, 4) & (1 << 1), 0); + } + + #[test] + fn write_one_to_clear_drops_the_line() { + let (mut uart, _capture) = uart_with_capture(); + uart.write(UCON, 4, u64::from(UCON_TXTHRESH_ENA)); + uart.write(UTXH, 4, u64::from(b'a')); + assert!(uart.irq_asserted()); + assert_ne!(uart.read(UTRSTAT, 4) & u64::from(UTRSTAT_TXTHRESH), 0); + + uart.write(UTRSTAT, 4, u64::from(UTRSTAT_TXTHRESH)); + assert!(!uart.irq_asserted()); + assert_eq!(uart.read(UTRSTAT, 4) & u64::from(UTRSTAT_TXTHRESH), 0); + } + + #[test] + fn enable_bit_masks_the_line_but_not_the_cause() { + let (mut uart, _capture) = uart_with_capture(); + uart.write(UTXH, 4, u64::from(b'a')); + assert!(!uart.irq_asserted(), "no enable bit, no line"); + assert_ne!(uart.read(UTRSTAT, 4) & u64::from(UTRSTAT_TXTHRESH), 0); + + uart.write(UCON, 4, u64::from(UCON_TXTHRESH_ENA)); + assert!(uart.irq_asserted(), "enabling reveals the pending cause"); + } + + #[test] + fn driver_tx_kick_completes_without_waiting_for_an_interrupt() { + // `enable_tx_pio`: set the enable bit, write the queued bytes inline, + // then `stop_tx` clears the enable bit. The line must end released. + let (mut uart, capture) = uart_with_capture(); + uart.write(UCON, 4, u64::from(UCON_TXTHRESH_ENA)); + for &byte in b"boot" { + uart.write(UTXH, 4, u64::from(byte)); + } + uart.write(UCON, 4, 0); + assert!(!uart.irq_asserted()); + assert_eq!(capture.collected(), b"boot"); + + // A later kick with a cause left over from before still makes progress: + // the handler clears the cause, then drains and disables the line. + uart.write(UCON, 4, u64::from(UCON_TXTHRESH_ENA)); + assert!(uart.irq_asserted()); + uart.write(UTRSTAT, 4, u64::from(UTRSTAT_TXTHRESH)); + uart.write(UTXH, 4, u64::from(b'!')); + uart.write(UCON, 4, 0); + assert!(!uart.irq_asserted()); + assert_eq!(capture.collected(), b"boot!"); + } + + #[test] + fn startup_registers_read_back() { + let (mut uart, _capture) = uart_with_capture(); + let ucon = 0x3a85_u32; // APPLE_S5L_UCON_DEFAULT with all four enable bits set + uart.write(ULCON, 4, 0x3); + uart.write(UCON, 4, u64::from(ucon)); + uart.write(UFCON, 4, u64::from(UFCON_STARTUP)); + uart.write(UMCON, 4, 0x10); + uart.write(UBRDIV, 4, 0x1a0); + uart.write(UFRACVAL, 4, 0x7); + + assert_eq!(uart.read(ULCON, 4), 0x3); + assert_eq!(uart.read(UCON, 4), u64::from(ucon)); + // The FIFO reset bits are self-clearing and do not read back. + assert_eq!(uart.read(UFCON, 4), u64::from(UFCON_STARTUP & !0b110)); + assert_eq!(uart.read(UMCON, 4), 0x10); + assert_eq!(uart.read(UBRDIV, 4), 0x1a0); + assert_eq!(uart.read(UFRACVAL, 4), 0x7); + } + + #[test] + fn unknown_offsets_read_zero_and_ignore_writes() { + let (mut uart, _capture) = uart_with_capture(); + uart.write(0x30, 4, 0xffff_ffff); + uart.write(0x38, 4, 0xffff_ffff); + assert_eq!(uart.read(0x30, 4), 0); + assert_eq!(uart.read(0x38, 4), 0); + assert!(!uart.irq_asserted()); + } + + #[test] + fn receive_fifo_drives_data_ready_and_urxh() { + let (mut uart, _capture) = uart_with_capture(); + assert_eq!(uart.read(URXH, 4), 0); + assert_eq!(uart.read(UTRSTAT, 4) & 1, 0); + + uart.feed_rx(b'k'); + assert_ne!(uart.read(UTRSTAT, 4) & 1, 0); + assert_eq!(uart.read(UFSTAT, 4) & 0xf, 1); + assert_eq!(uart.read(URXH, 4), u64::from(b'k')); + assert_eq!(uart.read(UTRSTAT, 4) & 1, 0); + } +} diff --git a/src/emulator/src/devices/virtio_blk.rs b/src/emulator/src/devices/virtio_blk.rs index bed07b9..ea827ca 100644 --- a/src/emulator/src/devices/virtio_blk.rs +++ b/src/emulator/src/devices/virtio_blk.rs @@ -1,5 +1,9 @@ -//! VirtIO-MMIO v2 Block Device (VIRTIO_ID_BLOCK). -//! Implements the VirtIO 1.0 (v2 modern) memory-mapped block device specification. +//! VirtIO block device: the request queue, shared by its transports. +//! +//! `VirtioBlock` is the VirtIO-MMIO v2 transport (the Linux virt board's `virtio,mmio` +//! device). `virtio_pci::VirtioBlockPci` is the same device behind the modern PCI transport. +//! Both own the register state and hand the queue to a [`BlockQueue`], which serves the +//! requests the driver made available. use crate::memory::{GuestMemory, SharedMemory}; use alloc::vec::Vec; @@ -9,10 +13,10 @@ pub const VERSION: u32 = 2; // Modern VirtIO v2 pub const DEVICE_ID_BLOCK: u32 = 2; // Block device pub const VENDOR_ID: u32 = 0x554d4551; // "QEMU" -// Feature bits +// Feature bits (virtio 1.2, section 5.2.3) pub const VIRTIO_BLK_F_RO: u64 = 1 << 5; pub const VIRTIO_BLK_F_BLK_SIZE: u64 = 1 << 6; -pub const VIRTIO_BLK_F_FLUSH: u64 = 1 << 11; +pub const VIRTIO_BLK_F_FLUSH: u64 = 1 << 9; pub const VIRTIO_F_VERSION_1: u64 = 1 << 32; // Request types @@ -30,6 +34,8 @@ pub const VRING_DESC_F_NEXT: u16 = 1; pub const VRING_DESC_F_WRITE: u16 = 2; pub const QUEUE_MAX_SIZE: u32 = 128; +/// Block addresses are in sectors of this many bytes. +pub const SECTOR_SIZE: u64 = 512; /// Storage backend for a block device. pub trait BlockBackend: Send { @@ -72,26 +78,206 @@ impl BlockBackend for Vec { Ok(()) } } + +/// The virtqueue a transport configures: its size, its three rings, and the progress +/// the device has made through them. #[derive(Clone, Copy, Default)] -struct Queue { - num: u32, - ready: bool, - desc_table: u64, - avail_ring: u64, - used_ring: u64, - last_avail_idx: u16, - last_used_idx: u16, +pub struct Queue { + pub num: u32, + pub ready: bool, + pub desc_table: u64, + pub avail_ring: u64, + pub used_ring: u64, + pub last_avail_idx: u16, + pub last_used_idx: u16, } -pub struct VirtioBlock { +/// The backing store, the guest memory the rings and buffers live in, and the one request +/// queue. Transports set `queue` from their registers and call [`BlockQueue::process`] +/// when the driver kicks it. +pub struct BlockQueue { pub backend: B, memory: SharedMemory, + pub queue: Queue, +} + +impl BlockQueue { + pub fn new(backend: B, memory: SharedMemory) -> Self { + Self { + backend, + memory, + queue: Queue::default(), + } + } + + /// The disk's capacity in sectors, as the device configuration reports it. + pub fn capacity_sectors(&self) -> u64 { + self.backend.len() / SECTOR_SIZE + } + + /// Serves every request the driver made available. Returns whether any was served. + pub fn process(&mut self) -> bool { + if self.queue.num == 0 || !self.queue.ready { + return false; + } + + let avail_addr = self.queue.avail_ring; + let desc_addr = self.queue.desc_table; + let used_addr = self.queue.used_ring; + let q_num = self.queue.num as u64; + + // Driver increments idx in avail ring + let avail_idx = self.read_guest_u16(avail_addr + 2); + let mut processed_any = false; + + while self.queue.last_avail_idx != avail_idx { + let ring_slot = (self.queue.last_avail_idx as u64) % q_num; + let head_idx = self.read_guest_u16(avail_addr + 4 + ring_slot * 2) as u64; + + // Walk descriptor chain + let mut curr = head_idx; + let mut req_header_addr = None; + let mut data_buffers: Vec<(u64, u32, bool)> = Vec::new(); + let mut status_addr = None; + + for _ in 0..q_num { + let d_addr = desc_addr + curr * 16; + let addr = self.read_guest_u64(d_addr); + let len = self.read_guest_u32(d_addr + 8); + let flags = self.read_guest_u16(d_addr + 12); + let next = self.read_guest_u16(d_addr + 14) as u64; + + let is_write = (flags & VRING_DESC_F_WRITE) != 0; + let has_next = (flags & VRING_DESC_F_NEXT) != 0; + + if req_header_addr.is_none() { + // First descriptor: virtio_blk_req header + req_header_addr = Some(addr); + } else if !has_next && is_write && len == 1 { + // Last descriptor in chain: status byte + status_addr = Some(addr); + } else { + // Data buffer + data_buffers.push((addr, len, is_write)); + } + + if !has_next { + break; + } + curr = next; + } + + let mut total_written = 0u32; + let mut status = VIRTIO_BLK_S_OK; + + if let Some(req_addr) = req_header_addr { + let req_type = self.read_guest_u32(req_addr); + let sector = self.read_guest_u64(req_addr + 8); + let mut disk_offset = sector.saturating_mul(SECTOR_SIZE); + + match req_type { + VIRTIO_BLK_T_IN => { + // Read from disk to guest memory + for (buf_addr, len, _) in data_buffers { + let mut tmp = alloc::vec![0u8; len as usize]; + if self.backend.read_at(disk_offset, &mut tmp).is_ok() { + let _ = self.memory.write(buf_addr, &tmp); + total_written = total_written.saturating_add(len); + } else { + status = VIRTIO_BLK_S_IOERR; + } + disk_offset = disk_offset.saturating_add(len as u64); + } + } + VIRTIO_BLK_T_OUT => { + // Write from guest memory to disk + for (buf_addr, len, _) in data_buffers { + let mut tmp = alloc::vec![0u8; len as usize]; + let _ = self.memory.read(buf_addr, &mut tmp); + if self.backend.write_at(disk_offset, &tmp).is_err() { + status = VIRTIO_BLK_S_IOERR; + } + disk_offset = disk_offset.saturating_add(len as u64); + } + } + VIRTIO_BLK_T_FLUSH => { + if self.backend.flush().is_err() { + status = VIRTIO_BLK_S_IOERR; + } + } + VIRTIO_BLK_T_GET_ID => { + for (buf_addr, len, _) in data_buffers { + let id = b"volt-blk\0"; + let copy_len = (len as usize).min(id.len()); + let _ = self.memory.write(buf_addr, &id[..copy_len]); + total_written = total_written.saturating_add(copy_len as u32); + } + } + _ => { + status = VIRTIO_BLK_S_IOERR; + } + } + } else { + status = VIRTIO_BLK_S_IOERR; + } + + // Write status byte (1 byte, written by device) + if let Some(s_addr) = status_addr { + let _ = self.memory.write(s_addr, &[status]); + total_written = total_written.saturating_add(1); + } + + // Write entry to used ring + let used_slot = (self.queue.last_used_idx as u64) % q_num; + let elem_addr = used_addr + 4 + used_slot * 8; + self.write_guest_u32(elem_addr, head_idx as u32); + self.write_guest_u32(elem_addr + 4, total_written); + + self.queue.last_used_idx = self.queue.last_used_idx.wrapping_add(1); + self.write_guest_u16(used_addr + 2, self.queue.last_used_idx); + + self.queue.last_avail_idx = self.queue.last_avail_idx.wrapping_add(1); + processed_any = true; + } + + processed_any + } + + fn read_guest_u16(&self, addr: u64) -> u16 { + let mut buf = [0u8; 2]; + let _ = self.memory.read(addr, &mut buf); + u16::from_le_bytes(buf) + } + + fn read_guest_u32(&self, addr: u64) -> u32 { + let mut buf = [0u8; 4]; + let _ = self.memory.read(addr, &mut buf); + u32::from_le_bytes(buf) + } + + fn read_guest_u64(&self, addr: u64) -> u64 { + let mut buf = [0u8; 8]; + let _ = self.memory.read(addr, &mut buf); + u64::from_le_bytes(buf) + } + + fn write_guest_u16(&mut self, addr: u64, val: u16) { + let _ = self.memory.write(addr, &val.to_le_bytes()); + } + + fn write_guest_u32(&mut self, addr: u64, val: u32) { + let _ = self.memory.write(addr, &val.to_le_bytes()); + } +} + +/// The VirtIO-MMIO v2 transport for the block device. +pub struct VirtioBlock { + core: BlockQueue, status: u32, device_features_sel: u32, driver_features_sel: u32, driver_features: u64, queue_sel: u32, - queue: Queue, interrupt_status: u32, line: Option, } @@ -99,14 +285,12 @@ pub struct VirtioBlock { impl VirtioBlock { pub fn new(backend: B, memory: SharedMemory) -> Self { Self { - backend, - memory, + core: BlockQueue::new(backend, memory), status: 0, device_features_sel: 0, driver_features_sel: 0, driver_features: 0, queue_sel: 0, - queue: Queue::default(), interrupt_status: 0, line: None, } @@ -116,14 +300,12 @@ impl VirtioBlock { impl VirtioBlock { pub fn with_line(self, line: L2) -> VirtioBlock { VirtioBlock { - backend: self.backend, - memory: self.memory, + core: self.core, status: self.status, device_features_sel: self.device_features_sel, driver_features_sel: self.driver_features_sel, driver_features: self.driver_features, queue_sel: self.queue_sel, - queue: self.queue, interrupt_status: self.interrupt_status, line: Some(line), } @@ -165,7 +347,7 @@ impl VirtioBlock { } } 0x044 => { - if self.queue_sel == 0 && self.queue.ready { + if self.queue_sel == 0 && self.core.queue.ready { 1 } else { 0 @@ -177,7 +359,7 @@ impl VirtioBlock { // Configuration space: // 0x100..0x108: capacity in 512-byte sectors (u64) 0x100..=0x107 => { - let capacity = self.backend.len() / 512; + let capacity = self.core.capacity_sectors(); let shift = (offset - 0x100) * 8; capacity >> shift } @@ -210,18 +392,18 @@ impl VirtioBlock { 0x030 => self.queue_sel = value as u32, 0x038 => { if self.queue_sel == 0 { - self.queue.num = (value as u32).min(QUEUE_MAX_SIZE); + self.core.queue.num = (value as u32).min(QUEUE_MAX_SIZE); } } 0x044 => { if self.queue_sel == 0 { - self.queue.ready = (value & 1) != 0; + self.core.queue.ready = (value & 1) != 0; } } 0x050 => { let queue_idx = value as u32; - if queue_idx == 0 && self.queue.ready { - self.process_queue(); + if queue_idx == 0 && self.core.queue.ready && self.core.process() { + self.raise_irq(); } } 0x064 => { @@ -234,183 +416,39 @@ impl VirtioBlock { self.status = value as u32; if self.status == 0 { // Reset - self.queue = Queue::default(); + self.core.queue = Queue::default(); self.lower_irq(); } } 0x080 => { - self.queue.desc_table = - (self.queue.desc_table & !0xffff_ffff) | (value & 0xffff_ffff) + self.core.queue.desc_table = + (self.core.queue.desc_table & !0xffff_ffff) | (value & 0xffff_ffff) + } + 0x084 => { + self.core.queue.desc_table = + (self.core.queue.desc_table & 0xffff_ffff) | (value << 32) } - 0x084 => self.queue.desc_table = (self.queue.desc_table & 0xffff_ffff) | (value << 32), 0x090 => { - self.queue.avail_ring = - (self.queue.avail_ring & !0xffff_ffff) | (value & 0xffff_ffff) + self.core.queue.avail_ring = + (self.core.queue.avail_ring & !0xffff_ffff) | (value & 0xffff_ffff) } - 0x094 => self.queue.avail_ring = (self.queue.avail_ring & 0xffff_ffff) | (value << 32), - 0x0a0 => { - self.queue.used_ring = (self.queue.used_ring & !0xffff_ffff) | (value & 0xffff_ffff) + 0x094 => { + self.core.queue.avail_ring = + (self.core.queue.avail_ring & 0xffff_ffff) | (value << 32) } - 0x0a4 => self.queue.used_ring = (self.queue.used_ring & 0xffff_ffff) | (value << 32), - _ => {} - } - } - - fn read_guest_u16(&self, addr: u64) -> u16 { - let mut buf = [0u8; 2]; - let _ = self.memory.read(addr, &mut buf); - u16::from_le_bytes(buf) - } - - fn read_guest_u32(&self, addr: u64) -> u32 { - let mut buf = [0u8; 4]; - let _ = self.memory.read(addr, &mut buf); - u32::from_le_bytes(buf) - } - - fn read_guest_u64(&self, addr: u64) -> u64 { - let mut buf = [0u8; 8]; - let _ = self.memory.read(addr, &mut buf); - u64::from_le_bytes(buf) - } - - fn write_guest_u16(&mut self, addr: u64, val: u16) { - let _ = self.memory.write(addr, &val.to_le_bytes()); - } - - fn write_guest_u32(&mut self, addr: u64, val: u32) { - let _ = self.memory.write(addr, &val.to_le_bytes()); - } - - fn process_queue(&mut self) { - if self.queue.num == 0 || !self.queue.ready { - return; - } - - let avail_addr = self.queue.avail_ring; - let desc_addr = self.queue.desc_table; - let used_addr = self.queue.used_ring; - let q_num = self.queue.num as u64; - - // Driver increments idx in avail ring - let avail_idx = self.read_guest_u16(avail_addr + 2); - let mut processed_any = false; - - while self.queue.last_avail_idx != avail_idx { - let ring_slot = (self.queue.last_avail_idx as u64) % q_num; - let head_idx = self.read_guest_u16(avail_addr + 4 + ring_slot * 2) as u64; - - // Walk descriptor chain - let mut curr = head_idx; - let mut req_header_addr = None; - let mut data_buffers: Vec<(u64, u32, bool)> = Vec::new(); - let mut status_addr = None; - - for _ in 0..q_num { - let d_addr = desc_addr + curr * 16; - let addr = self.read_guest_u64(d_addr); - let len = self.read_guest_u32(d_addr + 8); - let flags = self.read_guest_u16(d_addr + 12); - let next = self.read_guest_u16(d_addr + 14) as u64; - - let is_write = (flags & VRING_DESC_F_WRITE) != 0; - let has_next = (flags & VRING_DESC_F_NEXT) != 0; - - if req_header_addr.is_none() { - // First descriptor: virtio_blk_req header - req_header_addr = Some(addr); - } else if !has_next && is_write && len == 1 { - // Last descriptor in chain: status byte - status_addr = Some(addr); - } else { - // Data buffer - data_buffers.push((addr, len, is_write)); - } - - if !has_next { - break; - } - curr = next; - } - - let mut total_written = 0u32; - let mut status = VIRTIO_BLK_S_OK; - - if let Some(req_addr) = req_header_addr { - let req_type = self.read_guest_u32(req_addr); - let sector = self.read_guest_u64(req_addr + 8); - let mut disk_offset = sector.saturating_mul(512); - - match req_type { - VIRTIO_BLK_T_IN => { - // Read from disk to guest memory - for (buf_addr, len, _) in data_buffers { - let mut tmp = alloc::vec![0u8; len as usize]; - if self.backend.read_at(disk_offset, &mut tmp).is_ok() { - let _ = self.memory.write(buf_addr, &tmp); - total_written = total_written.saturating_add(len); - } else { - status = VIRTIO_BLK_S_IOERR; - } - disk_offset = disk_offset.saturating_add(len as u64); - } - } - VIRTIO_BLK_T_OUT => { - // Write from guest memory to disk - for (buf_addr, len, _) in data_buffers { - let mut tmp = alloc::vec![0u8; len as usize]; - let _ = self.memory.read(buf_addr, &mut tmp); - if self.backend.write_at(disk_offset, &tmp).is_err() { - status = VIRTIO_BLK_S_IOERR; - } - disk_offset = disk_offset.saturating_add(len as u64); - } - } - VIRTIO_BLK_T_FLUSH => { - if self.backend.flush().is_err() { - status = VIRTIO_BLK_S_IOERR; - } - } - VIRTIO_BLK_T_GET_ID => { - for (buf_addr, len, _) in data_buffers { - let id = b"volt-blk\0"; - let copy_len = (len as usize).min(id.len()); - let _ = self.memory.write(buf_addr, &id[..copy_len]); - total_written = total_written.saturating_add(copy_len as u32); - } - } - _ => { - status = VIRTIO_BLK_S_IOERR; - } - } - } else { - status = VIRTIO_BLK_S_IOERR; + 0x0a0 => { + self.core.queue.used_ring = + (self.core.queue.used_ring & !0xffff_ffff) | (value & 0xffff_ffff) } - - // Write status byte (1 byte, written by device) - if let Some(s_addr) = status_addr { - let _ = self.memory.write(s_addr, &[status]); - total_written = total_written.saturating_add(1); + 0x0a4 => { + self.core.queue.used_ring = + (self.core.queue.used_ring & 0xffff_ffff) | (value << 32) } - - // Write entry to used ring - let used_slot = (self.queue.last_used_idx as u64) % q_num; - let elem_addr = used_addr + 4 + used_slot * 8; - self.write_guest_u32(elem_addr, head_idx as u32); - self.write_guest_u32(elem_addr + 4, total_written); - - self.queue.last_used_idx = self.queue.last_used_idx.wrapping_add(1); - self.write_guest_u16(used_addr + 2, self.queue.last_used_idx); - - self.queue.last_avail_idx = self.queue.last_avail_idx.wrapping_add(1); - processed_any = true; - } - - if processed_any { - self.raise_irq(); + _ => {} } } } + pub trait VirtioIo { fn read(&mut self, offset: u64, size: u8) -> u64; fn write(&mut self, offset: u64, size: u8, value: u64); diff --git a/src/emulator/src/devices/virtio_pci.rs b/src/emulator/src/devices/virtio_pci.rs new file mode 100644 index 0000000..9747397 --- /dev/null +++ b/src/emulator/src/devices/virtio_pci.rs @@ -0,0 +1,535 @@ +//! VirtIO block device on PCI: the modern (virtio 1.0) transport. +//! +//! The device is a type-0 function, vendor 0x1af4, device 0x1042 (0x1040 plus the block +//! type). BAR0 holds the four virtio structures, each at its own offset, and the capability +//! list names them: +//! +//! | structure | offset | length | notes | +//! |-----------|--------|--------|------------------------------------| +//! | common | 0x0000 | 0x40 | feature, status, queue registers | +//! | notify | 0x1000 | 0x100 | a write of a queue index kicks it | +//! | ISR | 0x2000 | 0x4 | never raised: the firmware polls | +//! | device | 0x3000 | 0x100 | the block configuration | +//! +//! Requests are served by the same `BlockQueue` the MMIO transport uses. + +use super::pci::{ConfigSpace, PciFunction}; +use super::virtio_blk::{ + BlockBackend, BlockQueue, QUEUE_MAX_SIZE, Queue, SECTOR_SIZE, VIRTIO_BLK_F_BLK_SIZE, + VIRTIO_BLK_F_FLUSH, VIRTIO_F_VERSION_1, +}; +use crate::memory::SharedMemory; +use alloc::vec::Vec; + +const VIRTIO_VENDOR: u16 = 0x1af4; +/// The modern device ID: 0x1040 plus the virtio device type, 2 for block. +const DEVICE_ID_BLOCK: u16 = 0x1042; +/// The subsystem device ID carries the device type for transitional drivers. +const SUBSYSTEM_BLOCK: u16 = 2; +/// Base class 0x01, mass storage; subclass 0x80, other. +const CLASS_STORAGE_OTHER: u32 = 0x01_80_00; + +/// The BAR holding every structure, and its size. +const BAR: usize = 0; +const BAR_SIZE: u64 = 0x4000; + +const COMMON_OFFSET: u64 = 0x0000; +const COMMON_LEN: u64 = 0x40; +const NOTIFY_OFFSET: u64 = 0x1000; +const NOTIFY_LEN: u64 = 0x100; +/// Each queue's notify register sits at `queue_notify_off * NOTIFY_MULTIPLIER` into the region. +const NOTIFY_MULTIPLIER: u32 = 4; +const ISR_OFFSET: u64 = 0x2000; +const ISR_LEN: u64 = 0x4; +const DEVICE_OFFSET: u64 = 0x3000; +const DEVICE_LEN: u64 = 0x100; + +/// Capability identifier of a vendor-specific capability. +const CAP_VENDOR: u8 = 0x09; +const CFG_COMMON: u8 = 1; +const CFG_NOTIFY: u8 = 2; +const CFG_ISR: u8 = 3; +const CFG_DEVICE: u8 = 4; + +/// `MSI-X` vector meaning "none". +const NO_VECTOR: u16 = 0xffff; +const NUM_QUEUES: u16 = 1; +/// The features offered. Only the features this model implements are offered. +const FEATURES: u64 = VIRTIO_F_VERSION_1 | VIRTIO_BLK_F_BLK_SIZE | VIRTIO_BLK_F_FLUSH; + +/// Common configuration offsets, from virtio 1.2 section 4.1.4.3. +mod common { + pub const DEVICE_FEATURE_SELECT: u64 = 0x00; + pub const DEVICE_FEATURE: u64 = 0x04; + pub const DRIVER_FEATURE_SELECT: u64 = 0x08; + pub const DRIVER_FEATURE: u64 = 0x0c; + pub const MSIX_CONFIG: u64 = 0x10; + pub const DEVICE_STATUS: u64 = 0x14; + pub const QUEUE_SELECT: u64 = 0x16; + pub const QUEUE_SIZE: u64 = 0x18; + pub const QUEUE_MSIX_VECTOR: u64 = 0x1a; + pub const QUEUE_ENABLE: u64 = 0x1c; + pub const QUEUE_DESC: u64 = 0x20; + pub const QUEUE_DRIVER: u64 = 0x28; + pub const QUEUE_DEVICE: u64 = 0x30; +} + +/// A virtio block device on the modern PCI transport. +pub struct VirtioBlockPci { + config: ConfigSpace, + core: BlockQueue, + device_features_sel: u32, + driver_features_sel: u32, + driver_features: u64, + status: u8, + queue_select: u16, + msix_config: u16, + queue_msix_vector: u16, +} + +impl VirtioBlockPci { + pub fn new(backend: B, memory: SharedMemory) -> Self { + let mut config = ConfigSpace::new( + VIRTIO_VENDOR, + DEVICE_ID_BLOCK, + CLASS_STORAGE_OTHER, + VIRTIO_VENDOR, + SUBSYSTEM_BLOCK, + ); + config.set_bar(BAR, BAR_SIZE); + config.add_capability( + CAP_VENDOR, + &structure_body(CFG_COMMON, COMMON_OFFSET, COMMON_LEN, None), + ); + config.add_capability( + CAP_VENDOR, + &structure_body( + CFG_NOTIFY, + NOTIFY_OFFSET, + NOTIFY_LEN, + Some(NOTIFY_MULTIPLIER), + ), + ); + config.add_capability( + CAP_VENDOR, + &structure_body(CFG_ISR, ISR_OFFSET, ISR_LEN, None), + ); + config.add_capability( + CAP_VENDOR, + &structure_body(CFG_DEVICE, DEVICE_OFFSET, DEVICE_LEN, None), + ); + Self { + config, + core: BlockQueue::new(backend, memory), + device_features_sel: 0, + driver_features_sel: 0, + driver_features: 0, + status: 0, + queue_select: 0, + msix_config: NO_VECTOR, + queue_msix_vector: NO_VECTOR, + } + } + + fn reset(&mut self) { + self.device_features_sel = 0; + self.driver_features_sel = 0; + self.driver_features = 0; + self.queue_select = 0; + self.msix_config = NO_VECTOR; + self.queue_msix_vector = NO_VECTOR; + self.core.queue = Queue::default(); + } + + /// A 32-bit word of the common configuration, at a four-byte-aligned offset. + fn common_dword(&self, at: u64) -> u32 { + let queue = &self.core.queue; + let selected = self.queue_select == 0; + match at { + common::DEVICE_FEATURE_SELECT => self.device_features_sel, + common::DEVICE_FEATURE => match self.device_features_sel { + 0 => FEATURES as u32, + 1 => (FEATURES >> 32) as u32, + _ => 0, + }, + common::DRIVER_FEATURE_SELECT => self.driver_features_sel, + common::DRIVER_FEATURE => match self.driver_features_sel { + 0 => self.driver_features as u32, + 1 => (self.driver_features >> 32) as u32, + _ => 0, + }, + // msix_config, then num_queues. + 0x10 => u32::from(self.msix_config) | u32::from(NUM_QUEUES) << 16, + // device_status, config_generation (always 0), then queue_select. + 0x14 => u32::from(self.status) | u32::from(self.queue_select) << 16, + // queue_size (the maximum until the driver sets one), then queue_msix_vector. + 0x18 => { + let size = match (selected, queue.num) { + (false, _) => 0, + (true, 0) => QUEUE_MAX_SIZE, + (true, num) => num, + }; + size | u32::from(self.queue_msix_vector) << 16 + } + // queue_enable, then queue_notify_off, which is the queue index. + 0x1c => { + let enable = u32::from(selected && queue.ready); + enable | u32::from(self.queue_select) << 16 + } + common::QUEUE_DESC => queue.desc_table as u32, + 0x24 => (queue.desc_table >> 32) as u32, + common::QUEUE_DRIVER => queue.avail_ring as u32, + 0x2c => (queue.avail_ring >> 32) as u32, + common::QUEUE_DEVICE => queue.used_ring as u32, + 0x34 => (queue.used_ring >> 32) as u32, + _ => 0, + } + } + + /// Writes one field of the common configuration. Fields are written at their own + /// offsets; a 64-bit field arrives as one access or as two halves. + fn common_write(&mut self, at: u64, size: u8, value: u64) { + if size == 8 { + self.common_write(at, 4, value & 0xffff_ffff); + self.common_write(at + 4, 4, value >> 32); + return; + } + let low = value as u32; + match at { + common::DEVICE_FEATURE_SELECT => self.device_features_sel = low, + common::DRIVER_FEATURE_SELECT => self.driver_features_sel = low, + common::DRIVER_FEATURE => match self.driver_features_sel { + 0 => { + self.driver_features = (self.driver_features & !0xffff_ffff) | u64::from(low); + } + 1 => { + self.driver_features = + (self.driver_features & 0xffff_ffff) | u64::from(low) << 32; + } + _ => {} + }, + common::MSIX_CONFIG => self.msix_config = value as u16, + common::DEVICE_STATUS => { + self.status = value as u8; + if self.status == 0 { + self.reset(); + } + } + common::QUEUE_SELECT => self.queue_select = value as u16, + common::QUEUE_SIZE => { + if self.queue_select == 0 { + self.core.queue.num = (value as u32).min(QUEUE_MAX_SIZE); + } + } + common::QUEUE_MSIX_VECTOR => self.queue_msix_vector = value as u16, + common::QUEUE_ENABLE => { + if self.queue_select == 0 { + self.core.queue.ready = value & 1 != 0; + } + } + common::QUEUE_DESC => set_low(&mut self.core.queue.desc_table, low), + 0x24 => set_high(&mut self.core.queue.desc_table, low), + common::QUEUE_DRIVER => set_low(&mut self.core.queue.avail_ring, low), + 0x2c => set_high(&mut self.core.queue.avail_ring, low), + common::QUEUE_DEVICE => set_low(&mut self.core.queue.used_ring, low), + 0x34 => set_high(&mut self.core.queue.used_ring, low), + // Read-only fields, and the fields of queues this device does not have. + _ => {} + } + } + + /// A 32-bit word of the device configuration: the capacity in sectors, then the sector size. + fn device_dword(&self, at: u64) -> u32 { + match at { + 0x00 => self.core.capacity_sectors() as u32, + 0x04 => (self.core.capacity_sectors() >> 32) as u32, + 0x14 => SECTOR_SIZE as u32, + _ => 0, + } + } +} + +/// The body of a virtio vendor capability, after its identifier and `next` pointer. The +/// notify structure adds the multiplier that scales a queue's notify offset. +fn structure_body(cfg_type: u8, offset: u64, length: u64, multiplier: Option) -> Vec { + let total = if multiplier.is_some() { 20 } else { 16 }; + let mut body = Vec::with_capacity(total - 2); + body.push(total as u8); // cap_len + body.push(cfg_type); + body.push(BAR as u8); + body.push(0); // id, unused + body.extend_from_slice(&[0, 0]); // padding + body.extend_from_slice(&(offset as u32).to_le_bytes()); + body.extend_from_slice(&(length as u32).to_le_bytes()); + if let Some(multiplier) = multiplier { + body.extend_from_slice(&multiplier.to_le_bytes()); + } + body +} + +fn set_low(field: &mut u64, low: u32) { + *field = (*field & !0xffff_ffff) | u64::from(low); +} + +fn set_high(field: &mut u64, high: u32) { + *field = (*field & 0xffff_ffff) | u64::from(high) << 32; +} + +/// Reads `size` bytes at `offset` from a structure whose words `word` supplies, byte by byte. +fn read_bytes(offset: u64, size: u8, word: impl Fn(u64) -> u32) -> u64 { + (0..u64::from(size)).fold(0, |value, i| { + let at = offset + i; + let byte = (word(at & !3) >> ((at & 3) * 8)) as u8; + value | u64::from(byte) << (8 * i) + }) +} + +impl PciFunction for VirtioBlockPci { + fn config(&self) -> &ConfigSpace { + &self.config + } + + fn config_mut(&mut self) -> &mut ConfigSpace { + &mut self.config + } + + fn bar_read(&mut self, bar: usize, offset: u64, size: u8) -> u64 { + if bar != BAR { + return 0; + } + match offset { + o if o < COMMON_OFFSET + COMMON_LEN => { + read_bytes(o - COMMON_OFFSET, size, |at| self.common_dword(at)) + } + // The notify register is write-only, and the ISR is never raised. + o if (NOTIFY_OFFSET..NOTIFY_OFFSET + NOTIFY_LEN).contains(&o) + || (ISR_OFFSET..ISR_OFFSET + ISR_LEN).contains(&o) => + { + 0 + } + o if (DEVICE_OFFSET..DEVICE_OFFSET + DEVICE_LEN).contains(&o) => { + read_bytes(o - DEVICE_OFFSET, size, |at| self.device_dword(at)) + } + _ => 0, + } + } + + fn bar_write(&mut self, bar: usize, offset: u64, size: u8, value: u64) { + if bar != BAR { + return; + } + match offset { + o if o < COMMON_OFFSET + COMMON_LEN => { + self.common_write(o - COMMON_OFFSET, size, value); + } + // A driver kicks queue 0 with its index. The only queue this device has is 0. + o if (NOTIFY_OFFSET..NOTIFY_OFFSET + NOTIFY_LEN).contains(&o) + && value as u16 == 0 + && self.core.queue.ready => + { + self.core.process(); + } + // The device configuration is read only. + _ => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::devices::virtio_blk::{ + VIRTIO_BLK_S_OK, VIRTIO_BLK_T_IN, VRING_DESC_F_NEXT, VRING_DESC_F_WRITE, + }; + use crate::memory::{GuestMemory, PhysicalMemory, Region}; + + const RAM: u64 = 0x4000_0000; + const DESC: u64 = RAM + 0x1_0000; + const AVAIL: u64 = RAM + 0x1_1000; + const USED: u64 = RAM + 0x1_2000; + const HEADER: u64 = RAM + 0x2_0000; + const DATA: u64 = RAM + 0x2_1000; + const STATUS: u64 = RAM + 0x2_2000; + + fn device(disk: Vec) -> (VirtioBlockPci>, SharedMemory) { + let ram = Region::ram(RAM, alloc::vec![0u8; 1024 * 1024]).unwrap(); + let memory = SharedMemory::new(PhysicalMemory::new(alloc::vec![ram]).unwrap()); + (VirtioBlockPci::new(disk, memory.clone()), memory) + } + + /// The capabilities in list order, as `(structure, offset, length, multiplier)`. The + /// multiplier is read only for the notify structure, which has the longer capability. + fn capabilities(config: &ConfigSpace) -> Vec<(u8, u64, u64, Option)> { + let mut found = Vec::new(); + let mut at = config.read(0x34, 1) as usize; + while at != 0 { + assert_eq!(config.read(at, 1), u64::from(CAP_VENDOR)); + let cfg_type = config.read(at + 3, 1) as u8; + assert_eq!(config.read(at + 4, 1), 0, "every structure is in BAR0"); + let multiplier = (cfg_type == CFG_NOTIFY).then(|| config.read(at + 16, 4)); + found.push(( + cfg_type, + config.read(at + 8, 4), + config.read(at + 12, 4), + multiplier, + )); + at = config.read(at + 1, 1) as usize; + } + found + } + + #[test] + fn capabilities_locate_the_four_structures_in_bar0() { + let (dev, _) = device(alloc::vec![0; 512]); + let config = dev.config(); + assert_ne!(config.read(0x06, 2) & 0x10, 0, "status lists capabilities"); + assert_eq!( + capabilities(config), + [ + (CFG_COMMON, COMMON_OFFSET, COMMON_LEN, None), + ( + CFG_NOTIFY, + NOTIFY_OFFSET, + NOTIFY_LEN, + Some(u64::from(NOTIFY_MULTIPLIER)) + ), + (CFG_ISR, ISR_OFFSET, ISR_LEN, None), + (CFG_DEVICE, DEVICE_OFFSET, DEVICE_LEN, None), + ] + .to_vec() + ); + } + + fn write_descriptor( + memory: &mut SharedMemory, + index: u64, + addr: u64, + len: u32, + flags: u16, + next: u16, + ) { + let mut desc = [0u8; 16]; + desc[0..8].copy_from_slice(&addr.to_le_bytes()); + desc[8..12].copy_from_slice(&len.to_le_bytes()); + desc[12..14].copy_from_slice(&flags.to_le_bytes()); + desc[14..16].copy_from_slice(&next.to_le_bytes()); + memory.write(DESC + index * 16, &desc).unwrap(); + } + + /// Drives the device the way virtio-drivers does: reset, acknowledge, negotiate the + /// features, set FEATURES_OK, and configure and enable queue 0. + fn negotiate(dev: &mut VirtioBlockPci>) { + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1, 0); + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1, 0x3); + dev.bar_write(BAR, COMMON_OFFSET + common::DRIVER_FEATURE_SELECT, 4, 0); + dev.bar_write( + BAR, + COMMON_OFFSET + common::DRIVER_FEATURE, + 4, + VIRTIO_BLK_F_BLK_SIZE | VIRTIO_BLK_F_FLUSH, + ); + dev.bar_write(BAR, COMMON_OFFSET + common::DRIVER_FEATURE_SELECT, 4, 1); + dev.bar_write(BAR, COMMON_OFFSET + common::DRIVER_FEATURE, 4, 1); + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1, 0xb); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_SELECT, 2, 0); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_SIZE, 2, 16); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_DESC, 8, DESC); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_DRIVER, 8, AVAIL); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_DEVICE, 8, USED); + dev.bar_write(BAR, COMMON_OFFSET + common::QUEUE_ENABLE, 2, 1); + } + + #[test] + fn driver_negotiates_features_and_serves_a_read_through_the_pci_transport() { + let mut disk = alloc::vec![0x42u8; 2048]; + disk[512..1024].fill(0x37); // sector 1 is what the request reads + let (mut dev, mut memory) = device(disk); + + // The device offers VERSION_1 in the high word of its features. + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_FEATURE_SELECT, 4, 1); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::DEVICE_FEATURE, 4), + 1 + ); + + negotiate(&mut dev); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1), + 0xb, + "FEATURES_OK is kept when the driver's features are a subset of the device's" + ); + // The configured queue reads back: 8-byte fields come back whole. + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::QUEUE_DESC, 8), + DESC + ); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::QUEUE_ENABLE, 2), + 1 + ); + + // Read sector 1 into DATA: header, data (device-writable), status (device-writable). + let mut header = [0u8; 16]; + header[0..4].copy_from_slice(&VIRTIO_BLK_T_IN.to_le_bytes()); + header[8..16].copy_from_slice(&1u64.to_le_bytes()); + memory.write(HEADER, &header).unwrap(); + write_descriptor(&mut memory, 0, HEADER, 16, VRING_DESC_F_NEXT, 1); + write_descriptor( + &mut memory, + 1, + DATA, + 512, + VRING_DESC_F_WRITE | VRING_DESC_F_NEXT, + 2, + ); + write_descriptor(&mut memory, 2, STATUS, 1, VRING_DESC_F_WRITE, 0); + memory.write(STATUS, &[0xff]).unwrap(); + // Avail: idx 1, ring[0] = descriptor 0. + memory.write(AVAIL + 2, &1u16.to_le_bytes()).unwrap(); + memory.write(AVAIL + 4, &0u16.to_le_bytes()).unwrap(); + + dev.bar_write(BAR, NOTIFY_OFFSET, 2, 0); + + let mut data = [0u8; 512]; + memory.read(DATA, &mut data).unwrap(); + assert_eq!(data, [0x37u8; 512]); + let mut status = [0xffu8]; + memory.read(STATUS, &mut status).unwrap(); + assert_eq!(status[0], VIRTIO_BLK_S_OK); + let mut used_idx = [0u8; 2]; + memory.read(USED + 2, &mut used_idx).unwrap(); + assert_eq!( + u16::from_le_bytes(used_idx), + 1, + "the used ring records the request" + ); + } + + #[test] + fn the_device_configuration_reports_capacity_and_sector_size() { + let (mut dev, _) = device(alloc::vec![0; 2048]); + // Eight-byte capacity is one access, which the transport splits into two words. + assert_eq!(dev.bar_read(BAR, DEVICE_OFFSET, 8), 4); + assert_eq!(dev.bar_read(BAR, DEVICE_OFFSET + 0x14, 4), 512); + assert_eq!(dev.bar_read(BAR, DEVICE_OFFSET, 4), 4); + } + + #[test] + fn resetting_the_status_disables_the_queue() { + let (mut dev, _) = device(alloc::vec![0; 2048]); + negotiate(&mut dev); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::QUEUE_ENABLE, 2), + 1 + ); + dev.bar_write(BAR, COMMON_OFFSET + common::DEVICE_STATUS, 1, 0); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::QUEUE_ENABLE, 2), + 0 + ); + assert_eq!( + dev.bar_read(BAR, COMMON_OFFSET + common::DRIVER_FEATURE, 4), + 0 + ); + } +} diff --git a/src/emulator/src/memory.rs b/src/emulator/src/memory.rs index f98d324..e5e6d06 100644 --- a/src/emulator/src/memory.rs +++ b/src/emulator/src/memory.rs @@ -140,9 +140,12 @@ impl CodeTracker { } } -/// Host-visible RAM or a physical region without a host mapping. +/// Host-visible RAM, read-only storage such as flash, or a physical region without +/// a host mapping. Read-only storage serves reads and refuses writes; it has no +/// host pointer, so generated code's stores cannot bypass the refusal. pub enum Backing { Shared(Vec), + ReadOnly(Vec), Private, } @@ -165,6 +168,20 @@ impl Region { backing: Backing::Shared(bytes), }) } + + /// Read-only storage at `gpa`, such as a firmware image in flash. + pub fn rom(gpa: u64, bytes: Vec) -> Result { + let len = bytes.len() as u64; + gpa.checked_add(len).ok_or(MemoryError::AccessFault { + address: gpa, + length: bytes.len(), + })?; + Ok(Self { + gpa, + len, + backing: Backing::ReadOnly(bytes), + }) + } } /// Owns RAM storage. Every access must fit one physical region, as in Mirage. @@ -184,7 +201,7 @@ impl PhysicalMemory { address: region.gpa, length: usize::MAX, })?; - if let Backing::Shared(bytes) = ®ion.backing { + if let Backing::Shared(bytes) | Backing::ReadOnly(bytes) = ®ion.backing { if bytes.len() as u64 != region.len { return Err(MemoryError::AccessFault { address: region.gpa, @@ -228,7 +245,7 @@ impl PhysicalMemory { pub fn slice(&self, address: u64, length: usize) -> Result<&[u8], MemoryError> { let (index, offset) = self.find(address, length)?; match &self.regions[index].backing { - Backing::Shared(bytes) => bytes + Backing::Shared(bytes) | Backing::ReadOnly(bytes) => bytes .get(offset..offset + length) .ok_or(MemoryError::AccessFault { address, length }), Backing::Private => Err(MemoryError::PrivateMemory), @@ -237,14 +254,17 @@ impl PhysicalMemory { pub fn slice_mut(&mut self, address: u64, length: usize) -> Result<&mut [u8], MemoryError> { let (index, offset) = self.find(address, length)?; - // The caller writes through the returned slice, so it counts as a write - // already. The borrow (plus the RwLock for shared memory) keeps readers - // from observing the bumped version with the old bytes. - self.tracker.note_write(address, length); match &mut self.regions[index].backing { - Backing::Shared(bytes) => bytes - .get_mut(offset..offset + length) - .ok_or(MemoryError::AccessFault { address, length }), + Backing::Shared(bytes) => { + // The caller writes through the returned slice, so it counts as a write + // already. The borrow (plus the RwLock for shared memory) keeps readers + // from observing the bumped version with the old bytes. + self.tracker.note_write(address, length); + bytes + .get_mut(offset..offset + length) + .ok_or(MemoryError::AccessFault { address, length }) + } + Backing::ReadOnly(_) => Err(MemoryError::AccessFault { address, length }), Backing::Private => Err(MemoryError::PrivateMemory), } } @@ -302,7 +322,7 @@ impl SharedMemory { .iter() .filter_map(|region| match ®ion.backing { Backing::Shared(bytes) => Some((region.gpa, region.len, bytes.as_ptr().cast_mut())), - Backing::Private => None, + Backing::ReadOnly(_) | Backing::Private => None, }) .collect(); Self { @@ -424,6 +444,39 @@ mod tests { assert_eq!(memory.write(0x8000, b"x"), Err(MemoryError::PrivateMemory)); } + #[test] + fn read_only_regions_serve_reads_and_refuse_writes() { + let mut memory = + PhysicalMemory::new(vec![Region::rom(0, b"flash!".to_vec()).unwrap()]).unwrap(); + let mut out = [0u8; 5]; + memory.read(1, &mut out).unwrap(); + assert_eq!(&out, b"lash!"); + assert_eq!( + memory.write(0, b"x"), + Err(MemoryError::AccessFault { + address: 0, + length: 1 + }) + ); + assert_eq!(memory.slice(0, 6).unwrap(), b"flash!"); + } + + #[cfg(feature = "system")] + #[test] + fn read_only_regions_have_no_host_pointer_so_stores_cannot_bypass_them() { + let mut shared = SharedMemory::new( + PhysicalMemory::new(vec![Region::rom(0, vec![0xd5; 0x1000]).unwrap()]).unwrap(), + ); + assert_eq!(shared.host_page(0), None); + let mut out = [0u8; 4]; + shared.read(0, &mut out).unwrap(); + assert_eq!(out, [0xd5; 4]); + assert!(matches!( + shared.write(0, &[0]), + Err(MemoryError::AccessFault { .. }) + )); + } + #[cfg(feature = "system")] #[test] fn shared_memory_ram_access_is_lock_free_but_still_tracked() { diff --git a/src/emulator/src/system/boot.rs b/src/emulator/src/system/boot.rs index 1dd9a04..82a890d 100644 --- a/src/emulator/src/system/boot.rs +++ b/src/emulator/src/system/boot.rs @@ -43,6 +43,8 @@ pub enum Error { /// A boot option the selected kernel format cannot honour. Unsupported(&'static str), Memory(MemoryError), + /// The device tree the caller supplied cannot be patched. + DeviceTree(super::dtb::Error), } impl core::fmt::Display for Error { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { @@ -163,6 +165,65 @@ pub fn prepare_boot( }) } +/// Place a Linux `Image` at its text offset above `ram_base`, with `tree` (a device +/// tree the caller already built or patched) 2 MiB-aligned after it. The returned +/// layout's `boot_info` is the tree's address, which the kernel receives in `x0`. +pub fn place_with_tree( + memory: &mut impl GuestMemory, + image: &[u8], + tree: &[u8], + ram_base: u64, + ram_size: u64, +) -> Result { + let header = parse(image)?; + let entry = load_address(ram_base, header)?; + let occupied = header.image_size.max(image.len() as u64); + let tree_at = align(entry.checked_add(occupied).ok_or(Error::NoRoom)?)?; + let end = ram_base.checked_add(ram_size).ok_or(Error::NoRoom)?; + if tree_at + .checked_add(tree.len() as u64) + .is_none_or(|tree_end| tree_end > end) + { + return Err(Error::NoRoom); + } + memory.write(entry, image).map_err(Error::Memory)?; + memory.write(tree_at, tree).map_err(Error::Memory)?; + Ok(Layout { + entry, + boot_info: tree_at, + initrd: None, + }) +} + +/// Where a firmware image is mapped: flash at address 0, where the core resets. +pub const FIRMWARE_BASE: u64 = 0; + +/// Place a device tree at the base of RAM and enter the firmware at [`FIRMWARE_BASE`] with +/// `x0` zero. That is what QEMU's `-bios` does: its reset path sets only the PC, and the +/// tree goes to RAM base for a firmware that looks there. Without a tree in `x0` the +/// tinted-boot firmware assumes its built-in platform description, which places RAM up to +/// 4 GiB; passing the tree there would have it reserve memory it already owns. +pub fn prepare_firmware( + memory: &mut impl GuestMemory, + ram_base: u64, + ram_size: u64, +) -> Result { + let tree = super::fdt::build_firmware(ram_base, ram_size); + let end = ram_base.checked_add(ram_size).ok_or(Error::NoRoom)?; + if ram_base + .checked_add(tree.len() as u64) + .is_none_or(|tree_end| tree_end > end) + { + return Err(Error::NoRoom); + } + memory.write(ram_base, &tree).map_err(Error::Memory)?; + Ok(Layout { + entry: FIRMWARE_BASE, + boot_info: 0, + initrd: None, + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/emulator/src/system/dtb.rs b/src/emulator/src/system/dtb.rs new file mode 100644 index 0000000..90804fe --- /dev/null +++ b/src/emulator/src/system/dtb.rs @@ -0,0 +1,794 @@ +//! Flattened device tree patching for a device tree supplied by the user. +//! +//! The kernel reads `/chosen/bootargs` for its command line and `/memory` `reg` +//! for the RAM it may use, so both must describe what the emulator provides. The +//! blob is parsed into an owned tree of nodes and properties, keeping their order +//! and the memory reservation map, the two properties are edited there, and the +//! tree is serialized into a fresh blob. Nothing else in the tree is touched. + +use alloc::vec::Vec; +use core::fmt; + +const MAGIC: u32 = 0xd00d_feed; +const HEADER_LEN: usize = 40; +/// Newest format version this reader and writer speak. +const VERSION: u32 = 17; +const LAST_COMP_VERSION: u32 = 16; +const RESERVE_ENTRY_LEN: usize = 16; + +const FDT_BEGIN_NODE: u32 = 1; +const FDT_END_NODE: u32 = 2; +const FDT_PROP: u32 = 3; +const FDT_NOP: u32 = 4; +const FDT_END: u32 = 9; + +/// Cell counts the spec assumes when the root node omits them. +const DEFAULT_ADDRESS_CELLS: u32 = 2; +const DEFAULT_SIZE_CELLS: u32 = 1; + +const TRUNCATED_STRUCT: Error = Error::Malformed("structure block ends inside a token"); +const PROP_PAST_STRUCT: Error = Error::Malformed("property runs past structure block"); + +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + /// The blob is shorter than its header, or than the `totalsize` it declares. + Truncated { + len: usize, + needed: usize, + }, + BadMagic(u32), + UnsupportedVersion(u32), + /// The blob violates the flattened format; the message names the first problem found. + Malformed(&'static str), + /// No node named `memory*` with `device_type = "memory"` exists to receive RAM. + NoMemoryNode, + /// `#address-cells` or `#size-cells` is not 1 or 2, so `reg` cannot be encoded. + CellCount(u32), + /// The RAM base or size does not fit in the single cell the root node declares. + ValueTooLarge { + value: u64, + cells: u32, + }, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Truncated { len, needed } => { + write!(f, "device tree is {len} bytes, needs {needed}") + } + Error::BadMagic(magic) => write!(f, "bad device tree magic {magic:#010x}"), + Error::UnsupportedVersion(version) => { + write!( + f, + "unsupported device tree version {version}, need 17 or newer" + ) + } + Error::Malformed(what) => write!(f, "malformed device tree: {what}"), + Error::NoMemoryNode => { + write!( + f, + "device tree has no memory node with device_type \"memory\"" + ) + } + Error::CellCount(cells) => { + write!(f, "root cell count {cells} is not 1 or 2") + } + Error::ValueTooLarge { value, cells } => { + write!(f, "{value:#x} does not fit in {cells} cell(s)") + } + } + } +} + +impl core::error::Error for Error {} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Node { + name: Vec, + props: Vec, + children: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Prop { + name: Vec, + value: Vec, +} + +impl Node { + fn named(name: &[u8]) -> Self { + Node { + name: name.to_vec(), + props: Vec::new(), + children: Vec::new(), + } + } +} + +struct Parsed { + reservations: Vec<(u64, u64)>, + boot_cpuid_phys: u32, + root: Node, +} + +/// Return a new blob equal to `blob` except: if `bootargs` is `Some`, `/chosen` +/// gets the `bootargs` string property (replacing any existing one, creating +/// `/chosen` if absent); if `memory` is `Some((base, size))`, the first node +/// (depth-first) whose name starts with `memory` and whose `device_type` is +/// `memory` gets `reg` encoded with the root's `#address-cells` and `#size-cells`. +/// Every other node and property is preserved in order. +pub fn patch( + blob: &[u8], + bootargs: Option<&str>, + memory: Option<(u64, u64)>, +) -> Result, Error> { + let Parsed { + reservations, + boot_cpuid_phys, + mut root, + } = parse(blob)?; + if let Some(args) = bootargs { + let mut value = args.as_bytes().to_vec(); + value.push(0); + set_prop(chosen_node(&mut root), b"bootargs", value); + } + if let Some((base, size)) = memory { + let address_cells = cell_count(&root, b"#address-cells", DEFAULT_ADDRESS_CELLS)?; + let size_cells = cell_count(&root, b"#size-cells", DEFAULT_SIZE_CELLS)?; + let mut reg = Vec::new(); + push_cells(&mut reg, base, address_cells)?; + push_cells(&mut reg, size, size_cells)?; + let node = memory_node_mut(&mut root).ok_or(Error::NoMemoryNode)?; + set_prop(node, b"reg", reg); + } + serialize(&root, &reservations, boot_cpuid_phys) +} + +fn chosen_node(root: &mut Node) -> &mut Node { + let index = match root.children.iter().position(|c| c.name == b"chosen") { + Some(index) => index, + None => { + root.children.push(Node::named(b"chosen")); + root.children.len() - 1 + } + }; + &mut root.children[index] +} + +fn memory_node_mut(node: &mut Node) -> Option<&mut Node> { + if is_memory_device(node) { + return Some(node); + } + node.children.iter_mut().find_map(memory_node_mut) +} + +fn is_memory_device(node: &Node) -> bool { + node.name.starts_with(b"memory") + && prop_value(node, b"device_type") + .is_some_and(|value| value.strip_suffix(b"\0").unwrap_or(value) == b"memory") +} + +fn prop_value<'a>(node: &'a Node, name: &[u8]) -> Option<&'a [u8]> { + node.props + .iter() + .find(|p| p.name == name) + .map(|p| p.value.as_slice()) +} + +fn set_prop(node: &mut Node, name: &[u8], value: Vec) { + match node.props.iter_mut().find(|p| p.name == name) { + Some(prop) => prop.value = value, + None => node.props.push(Prop { + name: name.to_vec(), + value, + }), + } +} + +fn cell_count(root: &Node, name: &[u8], default: u32) -> Result { + match prop_value(root, name) { + None => Ok(default), + Some(value) => <[u8; 4]>::try_from(value) + .map(u32::from_be_bytes) + .map_err(|_| Error::Malformed("cell count property is not one u32")), + } +} + +/// Appends `value` as `cells` big-endian u32s; only 1 and 2 cells are valid. +fn push_cells(out: &mut Vec, value: u64, cells: u32) -> Result<(), Error> { + match cells { + 1 => { + let word = u32::try_from(value).map_err(|_| Error::ValueTooLarge { value, cells })?; + out.extend_from_slice(&word.to_be_bytes()); + Ok(()) + } + 2 => { + out.extend_from_slice(&value.to_be_bytes()); + Ok(()) + } + _ => Err(Error::CellCount(cells)), + } +} + +fn parse(blob: &[u8]) -> Result { + let header = parse_header(blob)?; + let data = blob + .get(..header.totalsize) + .ok_or(Error::Malformed("totalsize exceeds blob"))?; + let reservations = parse_reservations(data, header.off_mem_rsvmap)?; + let structs = block( + data, + header.off_dt_struct, + header.size_dt_struct, + "structure block lies outside totalsize", + )?; + let strings = block( + data, + header.off_dt_strings, + header.size_dt_strings, + "strings block lies outside totalsize", + )?; + if header.off_dt_struct % 4 != 0 { + return Err(Error::Malformed("structure block is not 4-byte aligned")); + } + let root = parse_struct(structs, strings)?; + Ok(Parsed { + reservations, + boot_cpuid_phys: header.boot_cpuid_phys, + root, + }) +} + +struct Header { + totalsize: usize, + off_dt_struct: u32, + off_dt_strings: u32, + off_mem_rsvmap: u32, + size_dt_strings: u32, + size_dt_struct: u32, + boot_cpuid_phys: u32, +} + +fn parse_header(blob: &[u8]) -> Result { + if blob.len() < HEADER_LEN { + return Err(Error::Truncated { + len: blob.len(), + needed: HEADER_LEN, + }); + } + let word = |at: usize| read_u32(blob, at).ok_or(Error::Malformed("header is truncated")); + let magic = word(0)?; + if magic != MAGIC { + return Err(Error::BadMagic(magic)); + } + let totalsize = word(4)? as usize; + let version = word(20)?; + if version < VERSION { + return Err(Error::UnsupportedVersion(version)); + } + if totalsize < HEADER_LEN { + return Err(Error::Malformed("totalsize is smaller than the header")); + } + if totalsize > blob.len() { + return Err(Error::Truncated { + len: blob.len(), + needed: totalsize, + }); + } + Ok(Header { + totalsize, + off_dt_struct: word(8)?, + off_dt_strings: word(12)?, + off_mem_rsvmap: word(16)?, + boot_cpuid_phys: word(28)?, + size_dt_strings: word(32)?, + size_dt_struct: word(36)?, + }) +} + +/// Reads (address, size) pairs up to the all-zero terminator. +fn parse_reservations(data: &[u8], off: u32) -> Result, Error> { + const PAST_END: Error = Error::Malformed("memory reservation block runs past totalsize"); + let mut entries = Vec::new(); + let mut pos = off as usize; + loop { + let address = read_u64(data, pos).ok_or(PAST_END)?; + let size = read_u64(data, pos + 8).ok_or(PAST_END)?; + pos += RESERVE_ENTRY_LEN; + if address == 0 && size == 0 { + return Ok(entries); + } + entries.push((address, size)); + } +} + +fn parse_struct(structs: &[u8], strings: &[u8]) -> Result { + let mut stack: Vec = Vec::new(); + let mut root: Option = None; + let mut pos = 0usize; + loop { + let token = read_u32(structs, pos).ok_or(TRUNCATED_STRUCT)?; + pos += 4; + match token { + FDT_BEGIN_NODE => { + if stack.is_empty() && root.is_some() { + return Err(Error::Malformed("more than one root node")); + } + let (name, next) = read_padded_name(structs, pos)?; + pos = next; + stack.push(Node { + name, + props: Vec::new(), + children: Vec::new(), + }); + } + FDT_END_NODE => { + let node = stack + .pop() + .ok_or(Error::Malformed("FDT_END_NODE with no open node"))?; + match stack.last_mut() { + Some(parent) => parent.children.push(node), + None => root = Some(node), + } + } + FDT_PROP => { + let len = read_u32(structs, pos).ok_or(TRUNCATED_STRUCT)? as usize; + let name_off = read_u32(structs, pos + 4).ok_or(TRUNCATED_STRUCT)?; + pos += 8; + let end = pos.checked_add(len).ok_or(PROP_PAST_STRUCT)?; + let value = structs.get(pos..end).ok_or(PROP_PAST_STRUCT)?.to_vec(); + pos = align4(end); + if pos > structs.len() { + return Err(PROP_PAST_STRUCT); + } + let name = string_at(strings, name_off)?; + let parent = stack + .last_mut() + .ok_or(Error::Malformed("property outside any node"))?; + parent.props.push(Prop { name, value }); + } + FDT_NOP => {} + FDT_END => { + if !stack.is_empty() { + return Err(Error::Malformed("FDT_END inside an open node")); + } + return root.ok_or(Error::Malformed("no root node before FDT_END")); + } + _ => return Err(Error::Malformed("unknown structure token")), + } + } +} + +/// Reads a NUL-terminated node name at `at`; returns it with the next token offset. +fn read_padded_name(structs: &[u8], at: usize) -> Result<(Vec, usize), Error> { + let rest = structs + .get(at..) + .ok_or(Error::Malformed("node name starts outside structure block"))?; + let nul = rest + .iter() + .position(|&b| b == 0) + .ok_or(Error::Malformed("unterminated node name"))?; + let next = align4(at + nul + 1); + if next > structs.len() { + return Err(Error::Malformed( + "node name padding runs past structure block", + )); + } + Ok((rest[..nul].to_vec(), next)) +} + +fn string_at(strings: &[u8], at: u32) -> Result, Error> { + let rest = strings.get(at as usize..).ok_or(Error::Malformed( + "property name offset outside strings block", + ))?; + let nul = rest + .iter() + .position(|&b| b == 0) + .ok_or(Error::Malformed("unterminated property name"))?; + Ok(rest[..nul].to_vec()) +} + +/// Serializes `root` into a version 17 blob. Property names are interned into the +/// strings block, so each distinct name is stored once. +fn serialize( + root: &Node, + reservations: &[(u64, u64)], + boot_cpuid_phys: u32, +) -> Result, Error> { + let mut writer = Writer { + structs: Vec::new(), + strings: Vec::new(), + names: Vec::new(), + }; + writer.node(root)?; + push_u32(&mut writer.structs, FDT_END); + + let off_mem_rsvmap = HEADER_LEN; + let rsvmap_len = (reservations.len() + 1) * RESERVE_ENTRY_LEN; + let off_dt_struct = off_mem_rsvmap + rsvmap_len; + let off_dt_strings = off_dt_struct + writer.structs.len(); + let totalsize = off_dt_strings + writer.strings.len(); + + let to_u32 = + |value: usize| u32::try_from(value).map_err(|_| Error::Malformed("blob exceeds 4 GiB")); + let mut out = Vec::with_capacity(totalsize); + push_u32(&mut out, MAGIC); + push_u32(&mut out, to_u32(totalsize)?); + push_u32(&mut out, to_u32(off_dt_struct)?); + push_u32(&mut out, to_u32(off_dt_strings)?); + push_u32(&mut out, to_u32(off_mem_rsvmap)?); + push_u32(&mut out, VERSION); + push_u32(&mut out, LAST_COMP_VERSION); + push_u32(&mut out, boot_cpuid_phys); + push_u32(&mut out, to_u32(writer.strings.len())?); + push_u32(&mut out, to_u32(writer.structs.len())?); + for &(address, size) in reservations { + out.extend_from_slice(&address.to_be_bytes()); + out.extend_from_slice(&size.to_be_bytes()); + } + out.extend_from_slice(&[0; RESERVE_ENTRY_LEN]); + out.extend_from_slice(&writer.structs); + out.extend_from_slice(&writer.strings); + Ok(out) +} + +struct Writer { + structs: Vec, + strings: Vec, + names: Vec<(Vec, u32)>, +} + +impl Writer { + fn name_offset(&mut self, name: &[u8]) -> Result { + if let Some((_, offset)) = self.names.iter().find(|(n, _)| n.as_slice() == name) { + return Ok(*offset); + } + let offset = u32::try_from(self.strings.len()) + .map_err(|_| Error::Malformed("strings block exceeds 4 GiB"))?; + self.strings.extend_from_slice(name); + self.strings.push(0); + self.names.push((name.to_vec(), offset)); + Ok(offset) + } + + fn node(&mut self, node: &Node) -> Result<(), Error> { + push_u32(&mut self.structs, FDT_BEGIN_NODE); + self.structs.extend_from_slice(&node.name); + self.structs.push(0); + pad4(&mut self.structs); + for prop in &node.props { + let name_off = self.name_offset(&prop.name)?; + let len = u32::try_from(prop.value.len()) + .map_err(|_| Error::Malformed("property value exceeds 4 GiB"))?; + push_u32(&mut self.structs, FDT_PROP); + push_u32(&mut self.structs, len); + push_u32(&mut self.structs, name_off); + self.structs.extend_from_slice(&prop.value); + pad4(&mut self.structs); + } + for child in &node.children { + self.node(child)?; + } + push_u32(&mut self.structs, FDT_END_NODE); + Ok(()) + } +} + +fn block<'a>(data: &'a [u8], off: u32, size: u32, what: &'static str) -> Result<&'a [u8], Error> { + let start = off as usize; + let end = start + .checked_add(size as usize) + .ok_or(Error::Malformed(what))?; + data.get(start..end).ok_or(Error::Malformed(what)) +} + +fn read_u32(bytes: &[u8], at: usize) -> Option { + let end = at.checked_add(4)?; + let word: [u8; 4] = bytes.get(at..end)?.try_into().ok()?; + Some(u32::from_be_bytes(word)) +} + +fn read_u64(bytes: &[u8], at: usize) -> Option { + let end = at.checked_add(8)?; + let word: [u8; 8] = bytes.get(at..end)?.try_into().ok()?; + Some(u64::from_be_bytes(word)) +} + +fn align4(value: usize) -> usize { + (value + 3) & !3 +} + +fn push_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_be_bytes()); +} + +fn pad4(out: &mut Vec) { + while !out.len().is_multiple_of(4) { + out.push(0); + } +} + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + + fn prop(name: &str, value: &[u8]) -> Prop { + Prop { + name: name.as_bytes().to_vec(), + value: value.to_vec(), + } + } + + fn cstr(s: &str) -> Vec { + let mut v = s.as_bytes().to_vec(); + v.push(0); + v + } + + fn be32(words: &[u32]) -> Vec { + words.iter().flat_map(|w| w.to_be_bytes()).collect() + } + + fn node(name: &str, props: Vec, children: Vec) -> Node { + Node { + name: name.as_bytes().to_vec(), + props, + children, + } + } + + /// Root with a memory-controller (not a memory device), memory, chosen and a serial. + fn fixture_with_cells(address_cells: u32, size_cells: u32) -> Node { + let mut root = Node::named(b""); + root.props = vec![ + prop("#address-cells", &be32(&[address_cells])), + prop("#size-cells", &be32(&[size_cells])), + prop("compatible", &cstr("apple,j274")), + ]; + root.children = vec![ + node( + "memory-controller@20e000000", + vec![prop("compatible", &cstr("apple,memory-controller"))], + vec![], + ), + node( + "memory@800000000", + vec![ + prop("device_type", &cstr("memory")), + prop("reg", &be32(&[8, 0, 2, 0])), + ], + vec![], + ), + node( + "chosen", + vec![prop("stdout-path", &cstr("/serial@235200000"))], + vec![], + ), + node( + "serial@235200000", + vec![ + prop("compatible", &cstr("ns16550")), + prop("reg", &be32(&[2, 0x3520_0000, 0, 0x4000])), + ], + vec![], + ), + ]; + root + } + + fn fixture() -> Node { + fixture_with_cells(2, 2) + } + + fn blob_of(root: &Node) -> Vec { + serialize(root, &[], 0).unwrap() + } + + fn tree_of(blob: &[u8]) -> Node { + parse(blob).unwrap().root + } + + fn child<'a>(node: &'a Node, name: &str) -> &'a Node { + node.children + .iter() + .find(|c| c.name == name.as_bytes()) + .unwrap() + } + + fn prop_of<'a>(node: &'a Node, name: &str) -> &'a [u8] { + prop_value(node, name.as_bytes()).unwrap() + } + + fn set_u32(blob: &mut [u8], at: usize, value: u32) { + blob[at..at + 4].copy_from_slice(&value.to_be_bytes()); + } + + #[test] + fn round_trip_keeps_tree_order_and_reservations() { + let original = fixture(); + let reservations = [(0x4000_0000u64, 0x0010_0000u64)]; + let blob = serialize(&original, &reservations, 0).unwrap(); + let out = patch(&blob, None, None).unwrap(); + let parsed = parse(&out).unwrap(); + assert_eq!(parsed.root, original); + assert_eq!(parsed.reservations, reservations); + } + + #[test] + fn bootargs_is_appended_to_chosen_when_absent() { + let out = patch( + &blob_of(&fixture()), + Some("console=ttyS0 root=/dev/vda"), + None, + ) + .unwrap(); + let out = tree_of(&out); + let chosen = child(&out, "chosen"); + let names: Vec<&[u8]> = chosen.props.iter().map(|p| p.name.as_slice()).collect(); + assert_eq!(names, [&b"stdout-path"[..], &b"bootargs"[..]]); + assert_eq!( + prop_of(chosen, "bootargs"), + b"console=ttyS0 root=/dev/vda\0" + ); + } + + #[test] + fn bootargs_replaces_existing_property_in_place() { + let mut root = fixture(); + let chosen = root + .children + .iter_mut() + .find(|c| c.name == b"chosen") + .unwrap(); + chosen.props.insert(0, prop("bootargs", &cstr("old"))); + let out = tree_of(&patch(&blob_of(&root), Some("new args"), None).unwrap()); + let chosen = child(&out, "chosen"); + let names: Vec<&[u8]> = chosen.props.iter().map(|p| p.name.as_slice()).collect(); + assert_eq!(names, [&b"bootargs"[..], &b"stdout-path"[..]]); + assert_eq!(prop_of(chosen, "bootargs"), b"new args\0"); + } + + #[test] + fn chosen_is_created_at_root_when_absent() { + let mut root = fixture(); + root.children.retain(|c| c.name != b"chosen"); + let out = tree_of(&patch(&blob_of(&root), Some("quiet"), None).unwrap()); + assert_eq!(out.children.len(), root.children.len() + 1); + assert_eq!(&out.children[..root.children.len()], &root.children[..]); + let chosen = out.children.last().unwrap(); + assert_eq!(chosen.name, b"chosen"); + assert_eq!(chosen.props, vec![prop("bootargs", b"quiet\0")]); + } + + #[test] + fn memory_reg_uses_two_two_cells() { + let out = patch( + &blob_of(&fixture()), + None, + Some((0x1_4000_0000, 0x2_0000_0000)), + ) + .unwrap(); + let out = tree_of(&out); + let mem = child(&out, "memory@800000000"); + assert_eq!(prop_of(mem, "reg"), be32(&[1, 0x4000_0000, 2, 0])); + } + + #[test] + fn memory_reg_uses_one_one_cells() { + let out = patch( + &blob_of(&fixture_with_cells(1, 1)), + None, + Some((0x4000_0000, 0x0800_0000)), + ) + .unwrap(); + let out = tree_of(&out); + let mem = child(&out, "memory@800000000"); + assert_eq!(prop_of(mem, "reg"), be32(&[0x4000_0000, 0x0800_0000])); + } + + #[test] + fn patch_touches_only_chosen_and_memory_reg() { + let original = fixture(); + let out = tree_of( + &patch( + &blob_of(&original), + Some("quiet"), + Some((0x1_4000_0000, 0x2_0000_0000)), + ) + .unwrap(), + ); + assert_eq!(out.props, original.props); + assert_eq!(out.name, original.name); + assert_eq!( + child(&out, "memory-controller@20e000000"), + child(&original, "memory-controller@20e000000") + ); + assert_eq!( + child(&out, "serial@235200000"), + child(&original, "serial@235200000") + ); + assert_eq!(out.children.len(), original.children.len()); + assert_eq!( + prop_of(child(&out, "memory@800000000"), "device_type"), + b"memory\0" + ); + assert_eq!( + prop_of(child(&out, "chosen"), "stdout-path"), + b"/serial@235200000\0" + ); + } + + #[test] + fn memory_value_too_large_for_one_cell_is_rejected() { + let blob = blob_of(&fixture_with_cells(1, 1)); + assert_eq!( + patch(&blob, None, Some((1 << 32, 0x1000))), + Err(Error::ValueTooLarge { + value: 1 << 32, + cells: 1 + }) + ); + } + + #[test] + fn unsupported_cell_count_is_rejected() { + let blob = blob_of(&fixture_with_cells(3, 2)); + assert_eq!( + patch(&blob, None, Some((0x4000_0000, 0x1000))), + Err(Error::CellCount(3)) + ); + } + + #[test] + fn memory_patch_needs_a_memory_device_node() { + let mut root = fixture(); + root.children.retain(|c| !c.name.starts_with(b"memory@")); + assert_eq!( + patch(&blob_of(&root), None, Some((0x4000_0000, 0x1000))), + Err(Error::NoMemoryNode) + ); + } + + #[test] + fn bad_magic_is_rejected() { + let mut blob = blob_of(&fixture()); + blob[0] ^= 0xff; + assert!(matches!(patch(&blob, None, None), Err(Error::BadMagic(_)))); + } + + #[test] + fn truncated_blob_is_rejected() { + let blob = blob_of(&fixture()); + assert!(matches!( + patch(&blob[..blob.len() - 1], None, None), + Err(Error::Truncated { .. }) + )); + assert!(matches!( + patch(&blob[..HEADER_LEN - 1], None, None), + Err(Error::Truncated { .. }) + )); + } + + #[test] + fn offsets_past_end_are_rejected() { + let blob = blob_of(&fixture()); + let total = blob.len() as u32; + + let mut bad_struct = blob.clone(); + set_u32(&mut bad_struct, 8, total + 4); + assert!(matches!( + patch(&bad_struct, None, None), + Err(Error::Malformed(_)) + )); + + let mut bad_strings = blob; + set_u32(&mut bad_strings, 12, u32::MAX); + assert!(matches!( + patch(&bad_strings, None, None), + Err(Error::Malformed(_)) + )); + } +} diff --git a/src/emulator/src/system/fdt.rs b/src/emulator/src/system/fdt.rs index 6efe45f..cb573fe 100644 --- a/src/emulator/src/system/fdt.rs +++ b/src/emulator/src/system/fdt.rs @@ -110,6 +110,11 @@ pub fn build(ram_base: u64, ram_size: u64, cmdline: &str) -> Vec { build_smp(ram_base, ram_size, cmdline, 1, None, false) } +/// Firmware discovers fw_cfg in the tree; direct kernel boots do not expose it. +pub fn build_firmware(ram_base: u64, ram_size: u64) -> Vec { + build_tree(ram_base, ram_size, "", 1, None, false, true) +} + pub fn build_smp( ram_base: u64, ram_size: u64, @@ -117,6 +122,18 @@ pub fn build_smp( cpus: u32, initrd: Option<(u64, u64)>, has_virtio: bool, +) -> Vec { + build_tree(ram_base, ram_size, cmdline, cpus, initrd, has_virtio, false) +} + +fn build_tree( + ram_base: u64, + ram_size: u64, + cmdline: &str, + cpus: u32, + initrd: Option<(u64, u64)>, + has_virtio: bool, + firmware: bool, ) -> Vec { let mut b = Builder::new(); b.begin(""); @@ -166,9 +183,22 @@ pub fn build_smp( b.cells("#address-cells", &[2]); b.cells("#size-cells", &[2]); b.property("interrupt-controller", &[]); - b.string("compatible", "arm,cortex-a15-gic"); - let mut registers = Vec::from(range(GICD_BASE, 0x1000)); - registers.extend_from_slice(&range(GIC_CPU_BASE, 0x2000)); + let registers = if firmware { + b.string("compatible", "arm,gic-v3"); + b.cells("#redistributor-regions", &[1]); + b.property("ranges", &[]); + let mut registers = Vec::from(range(GICD_BASE, crate::devices::gicv3::DISTRIBUTOR_SIZE)); + registers.extend_from_slice(&range( + super::afdt::GICR_BASE, + crate::devices::gicv3::REDISTRIBUTOR_SIZE, + )); + registers + } else { + b.string("compatible", "arm,cortex-a15-gic"); + let mut registers = Vec::from(range(GICD_BASE, 0x1000)); + registers.extend_from_slice(&range(GIC_CPU_BASE, 0x2000)); + registers + }; b.cells("reg", ®isters); b.cells("phandle", &[2]); b.end(); @@ -179,6 +209,57 @@ pub fn build_smp( b.cells("clocks", &[1, 1]); b.string("clock-names", "uartclk\0apb_pclk"); b.end(); + if firmware { + b.begin("fw-cfg@9020000"); + b.string("compatible", "qemu,fw-cfg-mmio"); + b.cells( + "reg", + &range(crate::devices::fw_cfg::BASE, crate::devices::fw_cfg::LEN), + ); + b.end(); + b.begin("pl031@9010000"); + b.string("compatible", "arm,pl031\0arm,primecell"); + b.cells( + "reg", + &range(crate::devices::pl031::BASE, crate::devices::pl031::LEN), + ); + b.cells("interrupts", &[0, 2, 4]); + b.cells("clocks", &[1]); + b.string("clock-names", "apb_pclk"); + b.end(); + b.begin("flash@0"); + b.string("compatible", "cfi-flash"); + b.cells("bank-width", &[4]); + let mut registers = Vec::from(range(0, crate::devices::nor_flash::LEN)); + registers.extend_from_slice(&range( + crate::devices::nor_flash::BASE, + crate::devices::nor_flash::LEN, + )); + b.cells("reg", ®isters); + b.end(); + b.begin("pcie@10000000"); + b.string("compatible", "pci-host-ecam-generic"); + b.string("device_type", "pci"); + b.cells("#address-cells", &[3]); + b.cells("#size-cells", &[2]); + b.cells("#interrupt-cells", &[1]); + b.cells("bus-range", &[0, 255]); + b.cells("linux,pci-domain", &[0]); + b.cells( + "reg", + &range( + crate::devices::pci::ECAM_BASE, + crate::devices::pci::ECAM_SIZE, + ), + ); + // Memory BAR space, matching QEMU virt. This host has no port-I/O devices. + b.cells( + "ranges", + &[0x0200_0000, 0, 0x1000_0000, 0, 0x1000_0000, 0, 0x2eff_0000], + ); + b.property("dma-coherent", &[]); + b.end(); + } if has_virtio { b.begin("virtio@a000000"); b.string("compatible", "virtio,mmio"); @@ -225,4 +306,33 @@ mod tests { assert!(tree.windows(17).any(|part| part == b"linux,initrd-end\0")); assert!(tree.windows(12).any(|part| part == b"virtio,mmio\0")); } + + #[test] + fn firmware_tree_describes_only_firmware_devices_and_gicv3() { + let tree = build_firmware(0x4000_0000, 4 << 30); + for compatible in [ + b"arm,gic-v3\0".as_slice(), + b"qemu,fw-cfg-mmio\0", + b"arm,pl031\0", + b"cfi-flash\0", + b"pci-host-ecam-generic\0", + ] { + assert!( + tree.windows(compatible.len()) + .any(|part| part == compatible) + ); + } + assert!(!tree.windows(17).any(|part| part == b"arm,cortex-a15-gic\0")); + assert!(!tree.windows(12).any(|part| part == b"virtio,mmio\0")); + assert!( + tree.windows(4) + .any(|part| part == (crate::devices::fw_cfg::BASE as u32).to_be_bytes()) + ); + let kernel_tree = build(0x4000_0000, 128 << 20, ""); + assert!( + !kernel_tree + .windows(16) + .any(|part| part == b"qemu,fw-cfg-mmio\0") + ); + } } diff --git a/src/emulator/src/system/m1.rs b/src/emulator/src/system/m1.rs new file mode 100644 index 0000000..1cc7af9 --- /dev/null +++ b/src/emulator/src/system/m1.rs @@ -0,0 +1,225 @@ +//! Apple M1 (t8103) SoC: the devices the Asahi Linux kernel's +//! `arch/arm64/boot/dts/apple/t8103.dtsi` describes, far enough to boot it. +//! +//! The addresses are the ones the M1 Mac mini's own device tree (`DeviceTree.j274ap` +//! in the macOS restore image) gives: arm-io's `ranges` places the SoC at +//! `0x2_0000_0000`, and the Linux nodes use the same absolute addresses. DRAM starts +//! at `0x8_0000_0000`. The kernel is booted with the device tree the caller supplies, +//! patched to describe the RAM this run provides. +use super::boot::{self, Error, Layout}; +use super::dtb; +use crate::devices::aic::{self, Aic}; +use crate::devices::pmgr::Pmgr; +use crate::devices::s5l_uart::{self, S5lUart}; +use crate::memory::GuestMemory; + +/// DRAM base (`memory@800000000` in the Asahi device tree). +pub const RAM_BASE: u64 = 0x8_0000_0000; +/// Default guest RAM: enough for the Linux kernel's own memory, well below the host's. +pub const DEFAULT_RAM_SIZE: usize = 1 << 30; +/// Interrupt controller (`aic: interrupt-controller@23b100000`). +pub const AIC_BASE: u64 = 0x2_3b10_0000; +/// Power-state register blocks (`power-management@…` with the `apple,pmgr` syscon): +/// the always-on block and the NUB block, as the device tree gives them. +pub const PMGR_BLOCKS: [(u64, u64); 2] = [(0x2_3b70_0000, 0x1_4000), (0x2_3d28_0000, 0x4000)]; +/// Console (`serial0: serial@235200000`), the `stdout-path` of the M1 device trees. +pub const UART_BASE: u64 = 0x2_3520_0000; +/// Hardware IRQ of the console (`AIC_IRQ 605` on `serial0`). +pub const UART_IRQ: u32 = 605; +/// Physical space below DRAM, where the device tree (an identity-mapped `simple-bus`) +/// places the SoC's devices. Devices this model does not implement live here. +pub const UNMODELED_BASE: u64 = 0; +pub const UNMODELED_SIZE: u64 = RAM_BASE; +/// `earlycon` without an address takes the device from `stdout-path`. +pub const DEFAULT_CMDLINE: &str = "earlycon nokaslr loglevel=8"; + +/// `MIDR_EL1` of an Icestorm core: implementer Apple (0x61), architecture from ID +/// registers (0xf), part 0x022. +pub const MIDR_EL1: u64 = 0x610f_0220; +/// `ID_AA64PFR0_EL1`: EL0, EL1 and EL2 in AArch64 (EL2 hosts the SPTM monitor, which +/// `CurrentEl` must report), no EL3, and floating point and Advanced SIMD implemented. +pub const ID_AA64PFR0_EL1: u64 = 0x111; +/// `ID_AA64MMFR0_EL1`: 4 KiB and 16 KiB translation granules (the Asahi kernel is built +/// for 16 KiB pages and refuses a CPU without one), no 64 KiB granule, 40-bit PA. +pub const ID_AA64MMFR0_EL1: u64 = (0xf << 24) | (1 << 20) | 2; +/// `ID_AA64ISAR1_EL1`: LRCPC, and the architected QARMA5 for address (APA) and generic +/// (GPA) pointer authentication. No implementation-defined algorithm (API, GPI). +pub const ID_AA64ISAR1_EL1: u64 = (1 << 20) | (1 << 24) | (1 << 4); + +/// The machine runs one vCPU. The device tree lists eight cores, but their +/// `cpu-release-addr` is zero, so Linux's spin-table code does not start them. +const CPUS: u32 = 1; + +/// Give `system` the identity of an M1 core. +pub fn identify(cpu: &mut crate::aarch64::cpu::Cpu) { + let system = &mut cpu.system; + system.midr_el1 = MIDR_EL1; + system.id_aa64pfr0_el1 = ID_AA64PFR0_EL1; + system.id_aa64mmfr0_el1 = ID_AA64MMFR0_EL1; + system.id_aa64isar1_el1 = ID_AA64ISAR1_EL1; + // The SPTM monitor polls `S3_6_c15_c12_4` until bit 0 is set before it continues + // (`mrs x0, s3_6_c15_c12_4; and x1, x0, #1; cbz x1, `). Nothing here names the + // register, so bit 0 reads as set, which is the only value that lets the loop exit. + cpu.apple_bank.0[volt_isa_aarch64::decode::apple_bank_slot(6, 12, 4)] = 1; +} + +/// The SoC devices on the bus: the AIC, the power-state blocks, the console, and the +/// line between the console and the AIC. +pub struct Soc { + aic: Aic, + pmgr: [Pmgr; PMGR_BLOCKS.len()], + uart: S5lUart, + unmodeled: u64, +} + +impl Soc { + /// `sink` receives every byte the kernel writes to the console. + pub fn new(sink: impl FnMut(u8) + Send + 'static) -> Self { + Self { + aic: Aic::new(CPUS), + pmgr: PMGR_BLOCKS.map(|(_, len)| Pmgr::new(len)), + uart: S5lUart::new(sink), + unmodeled: 0, + } + } + + /// Present the current device lines to the interrupt controller. Call before each + /// run of the CPU. + pub fn sync_lines(&mut self) { + self.aic.set_irq(UART_IRQ, self.uart.irq_asserted()); + } + + /// Whether the IRQ pin of CPU `cpu` is asserted. + pub fn irq_pending(&self, cpu: u32) -> bool { + self.aic.irq_pending(cpu) + } + + /// Accesses to arm-io devices this model does not implement: reads return zero and + /// writes are dropped, so a probe of such a device cannot fault the kernel. + pub fn unmodeled_accesses(&self) -> u64 { + self.unmodeled + } + + /// The power-state block and offset an access of `size` bytes at `address` falls in. + fn pmgr_window(address: u64, size: u8) -> Option<(usize, u64)> { + PMGR_BLOCKS + .iter() + .enumerate() + .find_map(|(block, &(base, len))| { + window(address, size, base, len).map(|offset| (block, offset)) + }) + } + + /// Read `size` bytes at guest-physical `address`, or `None` when no SoC device maps it. + pub fn read(&mut self, cpu: u32, address: u64, size: u8) -> Option { + if let Some(offset) = window(address, size, AIC_BASE, aic::LEN) { + return Some(self.aic.read(cpu, offset, size)); + } + if let Some((block, offset)) = Self::pmgr_window(address, size) { + return Some(self.pmgr[block].read(offset, size)); + } + if let Some(offset) = window(address, size, UART_BASE, s5l_uart::LEN) { + return Some(self.uart.read(offset, size)); + } + if window(address, size, UNMODELED_BASE, UNMODELED_SIZE).is_some() { + self.unmodeled += 1; + return Some(0); + } + None + } + + /// Write `size` bytes to guest-physical `address`; `false` when no SoC device maps it. + pub fn write(&mut self, cpu: u32, address: u64, size: u8, value: u64) -> bool { + if let Some(offset) = window(address, size, AIC_BASE, aic::LEN) { + self.aic.write(cpu, offset, size, value); + return true; + } + if let Some((block, offset)) = Self::pmgr_window(address, size) { + self.pmgr[block].write(offset, size, value); + return true; + } + if let Some(offset) = window(address, size, UART_BASE, s5l_uart::LEN) { + self.uart.write(offset, size, value); + return true; + } + if window(address, size, UNMODELED_BASE, UNMODELED_SIZE).is_some() { + self.unmodeled += 1; + return true; + } + false + } +} + +/// Offset of an access of `size` bytes at `address` within `[base, base + len)`. +/// An access that straddles the end of the window matches nothing. +fn window(address: u64, size: u8, base: u64, len: u64) -> Option { + let end = address.checked_add(u64::from(size))?; + (address >= base && end <= base + len).then(|| address - base) +} + +/// Place a Linux `Image` in RAM and the device tree after it, with the tree's +/// `/chosen/bootargs` set to `cmdline` (when non-empty) and its memory node describing +/// `ram_size` bytes at [`RAM_BASE`]. The kernel is entered with `x0` = the tree. +pub fn prepare( + memory: &mut impl GuestMemory, + image: &[u8], + device_tree: &[u8], + cmdline: &str, + ram_size: u64, +) -> Result { + let bootargs = (!cmdline.is_empty()).then_some(cmdline); + let tree = + dtb::patch(device_tree, bootargs, Some((RAM_BASE, ram_size))).map_err(Error::DeviceTree)?; + boot::place_with_tree(memory, image, &tree, RAM_BASE, ram_size) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn window_matches_only_accesses_entirely_inside_the_device() { + assert_eq!(window(AIC_BASE, 4, AIC_BASE, aic::LEN), Some(0)); + assert_eq!( + window(AIC_BASE + aic::LEN - 4, 4, AIC_BASE, aic::LEN), + Some(aic::LEN - 4) + ); + // Straddling the end, or wrapping, matches nothing. + assert_eq!(window(AIC_BASE + aic::LEN - 2, 4, AIC_BASE, aic::LEN), None); + assert_eq!(window(u64::MAX - 1, 4, AIC_BASE, aic::LEN), None); + } + + #[test] + fn soc_routes_by_address_and_declines_everything_else() { + let mut soc = Soc::new(|_| {}); + // The AIC identifies the reading CPU at WHOAMI. + assert_eq!(soc.read(0, AIC_BASE + 0x2000, 4), Some(0)); + // Above DRAM, where the SoC maps nothing, is not an M1 address. + assert_eq!(soc.read(0, RAM_BASE + (1 << 40), 4), None); + assert!(!soc.write(0, RAM_BASE + (1 << 40), 4, 0)); + assert_eq!(soc.unmodeled_accesses(), 0); + } + + #[test] + fn power_state_registers_of_each_pmgr_block_acknowledge_their_target() { + let mut soc = Soc::new(|_| {}); + for (base, _) in PMGR_BLOCKS { + // A domain's state register, as its driver polls it. + let state = base + 0x100; + assert_eq!(soc.read(0, state, 4), Some(0xff)); + assert!(soc.write(0, state, 4, 0x0)); + assert_eq!(soc.read(0, state, 4), Some(0x00)); + } + assert_eq!(soc.unmodeled_accesses(), 0); + } + + #[test] + fn unmodeled_arm_io_device_reads_zero_and_drops_writes_and_is_counted() { + let mut soc = Soc::new(|_| {}); + // The pin controller (`pinctrl@23e820000`) is probed at boot. + assert_eq!(soc.read(0, 0x2_3e82_0000, 4), Some(0)); + assert!(soc.write(0, 0x2_3e82_0000, 4, 0xffff_ffff)); + assert_eq!(soc.read(0, 0x2_3e82_0000, 4), Some(0)); + assert_eq!(soc.unmodeled_accesses(), 3); + } +} diff --git a/src/emulator/src/system/machine.rs b/src/emulator/src/system/machine.rs index 5abff51..49ff311 100644 --- a/src/emulator/src/system/machine.rs +++ b/src/emulator/src/system/machine.rs @@ -2,12 +2,12 @@ use super::afdt; use super::fdt; use crate::aarch64::cpu::{Exclusive, Trap}; -use crate::aarch64::decode::SignExtend; use crate::aarch64::simd_struct; use crate::aarch64::{Cache, Cpu}; use crate::aarch64::{exception, translate}; use crate::devices::{bus::Device, gicv2::Gicv2, pl011::Pl011}; use crate::memory::{GuestMemory, MemoryError}; +use volt_isa_aarch64::decode::SignExtend; #[derive(Debug)] pub enum Error { @@ -44,6 +44,33 @@ pub trait DeviceIo { fn write_virtio(&mut self, offset: u64, size: u8, value: u64) { let _ = (offset, size, value); } + /// A board-specific device (SoC peripherals or firmware devices). `None` + /// declines the access, which then goes to PCI and the fixed windows below. + fn read_board(&mut self, cpu_id: u32, address: u64, size: u8) -> Option { + let _ = (cpu_id, address, size); + None + } + /// Returns `false` when the board's devices do not claim the address. + fn write_board(&mut self, cpu_id: u32, address: u64, size: u8, value: u64) -> bool { + let _ = (cpu_id, address, size, value); + false + } + /// A read of an address the PCI bus claims: configuration space, or a memory BAR of a + /// function that decodes memory. `None` declines the access. + fn read_pci(&mut self, address: u64, size: u8) -> Option { + let _ = (address, size); + None + } + /// Returns `false` when no PCI function claims the address. + fn write_pci(&mut self, address: u64, size: u8, value: u64) -> bool { + let _ = (address, size, value); + false + } + /// Whether the fixed windows below (the QEMU `virt` UART, GIC and virtio) exist on + /// this board. A board that maps none of them makes accesses there fault. + fn has_fixed_windows(&self) -> bool { + true + } } impl DeviceIo for (&mut Pl011, &mut Gicv2) { @@ -167,6 +194,10 @@ pub struct Machine { /// Set when a GICv3 forwards PPI 27 (the virtual timer) as a group 0 FIQ: the /// priority it was given. `None` leaves the timer to the GICv2 path. pub vtimer_fiq: Option, + /// A FIQ source the board drives directly. The M1 interrupt controller presents + /// the virtual timer this way: it is level-high while the timer's own condition + /// holds, with no GIC-style priority gating. + pub fiq_line: bool, timer_fired: bool, pub stalled_at: u64, pub stalled: Option, @@ -175,6 +206,92 @@ pub struct Machine { pub blocks_run: u64, faulted: bool, device_access: bool, + /// Per-instruction observer installed by [`Machine::set_step_trace`]. + step_trace: Option>, + /// Per-access observer installed by [`Machine::set_memory_trace`]. + memory_trace: Option>, + /// The step in progress: its start `pc` and the system registers before it ran. + step_pending: Option<(u64, crate::aarch64::cpu::System)>, +} + +/// One step seen by a [`Machine::set_step_trace`] observer. Single-stepping makes each +/// step one instruction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Step { + /// Address the step started at. + pub pc: u64, + /// Address after the step. + pub next_pc: u64, + /// Each watched system register that changed, as `(name, before, after)`. + pub changes: Vec<(&'static str, u64, u64)>, +} + +/// One load or store seen by a [`Machine::set_memory_trace`] observer, after translation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MemoryAccess { + /// Address of the instruction that made the access. + pub pc: u64, + pub virtual_address: u64, + pub physical: u64, + /// Bytes accessed: 1, 2, 4, 8 or 16. + pub width: u8, + pub write: bool, + /// The low 8 bytes stored; 0 for a load, whose value is not known before it runs. + pub value: u64, +} + +/// The watched system registers whose values differ between `before` and `after`. +fn system_changes( + before: &crate::aarch64::cpu::System, + after: &crate::aarch64::cpu::System, +) -> Vec<(&'static str, u64, u64)> { + macro_rules! watch { + ($($field:ident),* $(,)?) => {{ + let mut changes = Vec::new(); + $( + if before.$field != after.$field { + changes.push((stringify!($field), before.$field as u64, after.$field as u64)); + } + )* + changes + }}; + } + let mut changes = watch!( + el, + spsel, + daif, + pan, + spsr_el1, + elr_el1, + esr_el1, + far_el1, + vbar_el1, + sctlr_el1, + ttbr0_el1, + ttbr1_el1, + tcr_el1, + mair_el1, + spsr_el2, + elr_el2, + esr_el2, + far_el2, + vbar_el2, + sctlr_el2, + ttbr0_el2, + ttbr1_el2, + tcr_el2, + mair_el2, + hcr_el2, + tpidr_el2, + cntvoff_el2, + ); + const SP_NAMES: [&str; 4] = ["sp_el0", "sp_el1", "sp_el2", "sp_el3"]; + for (index, name) in SP_NAMES.iter().enumerate() { + if before.sp_el[index] != after.sp_el[index] { + changes.push((name, before.sp_el[index], after.sp_el[index])); + } + } + changes } impl Machine { pub fn new(memory: M) -> Self { @@ -201,6 +318,7 @@ impl Machine { watch_epoch: 0, irq_line: false, vtimer_fiq: None, + fiq_line: false, timer_fired: false, stalled_at: 0, stalled: None, @@ -209,6 +327,9 @@ impl Machine { blocks_run: 0, faulted: false, device_access: false, + step_trace: None, + memory_trace: None, + step_pending: None, } } /// A vCPU whose translated blocks are shared with other vCPUs. @@ -260,16 +381,69 @@ impl Machine { && self.timer_due() }) } + /// Installs a per-instruction trace. While set, blocks are compiled one instruction at + /// a time, and `sink` sees each step with the watched system registers it changed. + /// `None` removes the trace. + pub fn set_step_trace(&mut self, sink: Option>) { + self.step_trace = sink; + self.rebuild_cache(); + } + /// Installs a per-access trace: `sink` sees every load and store the guest makes, + /// translated, except atomics and SIMD structure accesses. While set, the inline data + /// TLB is off so every access reaches the host. `None` removes the trace. + pub fn set_memory_trace(&mut self, sink: Option>) { + self.memory_trace = sink; + self.rebuild_cache(); + } + /// A fresh block cache for the trace state: single-step blocks while stepping, no inline + /// data TLB while either trace is set, and the default cache otherwise. Compiled blocks + /// and background compilation of the previous cache are dropped. + fn rebuild_cache(&mut self) { + let traced = self.step_trace.is_some() || self.memory_trace.is_some(); + let inline = !traced && !std::env::var_os("VOLT_INLINE_MEMORY").is_some_and(|v| v == "0"); + self.cache = if self.step_trace.is_some() { + Cache::single_step(false) + } else if inline { + Cache::with_inline_memory() + } else { + Cache::new() + }; + self.step_pending = None; + } pub fn run(&mut self, serial: &mut Pl011, gic: &mut Gicv2) -> Result { self.run_with_devices(&mut (serial, gic)) } + /// Reports the step in progress, if any, with the registers it changed. A step ends + /// when the next one starts or the run returns, so an exception entry the step + /// caused is part of it. + fn flush_step(&mut self) { + if let Some((pc, before)) = self.step_pending.take() + && let Some(sink) = self.step_trace.as_mut() + { + sink(&Step { + pc, + next_pc: self.cpu.pc, + changes: system_changes(&before, &self.cpu.system), + }); + } + } pub fn run_with_devices(&mut self, devices: &mut impl DeviceIo) -> Result { + let result = self.run_inner(devices); + self.flush_step(); + result + } + fn run_inner(&mut self, devices: &mut impl DeviceIo) -> Result { self.stalled = None; for _ in 0..64 { if self.timer_edge() { return Ok(Exit::Timer); } - let fiq = self.fiq_asserted(); + self.flush_step(); + self.step_pending = self + .step_trace + .is_some() + .then(|| (self.cpu.pc, self.cpu.system.clone())); + let fiq = self.fiq_line || self.fiq_asserted(); self.cpu.system.isr_el1 = u64::from(self.irq_line) << 7 | u64::from(fiq) << 6; let kind = if fiq && self.cpu.system.daif & 1 == 0 { Some(exception::Kind::Fiq) @@ -312,7 +486,7 @@ impl Machine { continue; } if let crate::aarch64::compile::Error::Decode { word, pc } = &error - && crate::aarch64::decode::is_undefined(*word) + && volt_isa_aarch64::decode::is_undefined(*word) { // An Undefined Instruction exception (EC 0, 32-bit instruction). self.cpu.pc = *pc; @@ -384,14 +558,35 @@ impl Machine { self.cpu.trap = Trap::None; exception::eret(&mut self.cpu); } - Trap::Svc | Trap::Brk => { - let ec = if self.cpu.trap == Trap::Svc { 0x15 } else { 0 }; + Trap::Svc => { + self.cpu.trap = Trap::None; + let pc = self.cpu.pc; + // The SVC immediate goes in ISS, as a BRK's does. + let immediate = self.cpu.address as u32; + exception::take( + &mut self.cpu, + exception::Reason { + ec: 0x15, + status: 0, + instruction: true, + iss: immediate, + ..Default::default() + }, + pc, + ); + } + Trap::Brk => { self.cpu.trap = Trap::None; let pc = self.cpu.pc; + let immediate = self.cpu.address as u32; + // A breakpoint's syndrome is EC 0x3c with its immediate in ISS. exception::take( &mut self.cpu, exception::Reason { - ec, + ec: 0x3c, + status: 0, + instruction: true, + iss: immediate, ..Default::default() }, pc, @@ -407,12 +602,12 @@ impl Machine { ec: 0, instruction: true, status: 0, + iss: 0, }, pc, ); } Trap::Isb => { - self.tlb.purge_faults(); self.cpu.trap = Trap::None; } Trap::Tlbi => { @@ -613,17 +808,22 @@ impl Machine { } } fn abort(&mut self, address: u64, permission: bool, instruction: bool, write: bool) { - let ec = if instruction { - if self.cpu.system.el == 1 { 0x21 } else { 0x20 } - } else if self.cpu.system.el == 1 { - 0x25 - } else { - 0x24 + let status = match (permission, instruction) { + (false, _) => exception::status::TRANSLATION, + (true, false) => exception::status::PERMISSION, + (true, true) => exception::status::PERMISSION_FETCH, }; - let status = if permission { - if instruction { 13 } else { 12 } - } else { - 4 + self.abort_with_status(address, status, instruction, write); + } + /// A synchronous abort with fault status `status`. + fn abort_with_status(&mut self, address: u64, status: u8, instruction: bool, write: bool) { + // EC 0x20/0x24 for a lower-EL abort, 0x21/0x25 for one taken from EL1 or EL2. + let same_el = self.cpu.system.el != 0; + let ec = match (instruction, same_el) { + (true, false) => 0x20, + (true, true) => 0x21, + (false, false) => 0x24, + (false, true) => 0x25, }; self.cpu.trap = Trap::None; if !instruction { @@ -636,6 +836,7 @@ impl Machine { ec, instruction, status, + iss: 0, }, address, ); @@ -908,6 +1109,16 @@ impl Machine { return Ok(()); } }; + if let Some(sink) = self.memory_trace.as_mut() { + sink(&MemoryAccess { + pc: self.cpu.pc.wrapping_sub(4), + virtual_address: address, + physical, + width, + write: !load, + value: if load { 0 } else { value }, + }); + } let mut bytes = [0u8; 16]; if !load { if self.cpu.vector_dest { @@ -963,7 +1174,7 @@ impl Machine { }; if matches!(result, Err(MemoryError::AccessFault { .. })) { self.unmapped += 1; - self.abort(address, false, false, !load); + self.abort_with_status(address, exception::status::SYNC_EXTERNAL, false, !load); return Ok(()); } result @@ -973,9 +1184,28 @@ impl Machine { Err(MemoryError::AccessFault { .. }) => { self.device_access = true; let end = physical.checked_add(width as u64); - let window = - |base, length| physical >= base && end.is_some_and(|end| end <= base + length); - let read = if window(fdt::UART_BASE, crate::devices::pl011::LEN) { + let fixed = devices.has_fixed_windows(); + let window = |base, length| { + fixed && physical >= base && end.is_some_and(|end| end <= base + length) + }; + let board = if load { + devices.read_board(self.cpu_id, physical, width).map(Some) + } else { + devices + .write_board(self.cpu_id, physical, width, value) + .then_some(None) + }; + // Only accesses the board's own devices decline reach the PCI bus. + let pci = match (&board, load) { + (Some(_), _) => None, + (None, true) => devices.read_pci(physical, width).map(Some), + (None, false) => devices.write_pci(physical, width, value).then_some(None), + }; + let read = if let Some(read) = board { + read + } else if let Some(read) = pci { + read + } else if window(fdt::UART_BASE, crate::devices::pl011::LEN) { if load { Some(devices.read_uart(physical - fdt::UART_BASE, width)) } else { @@ -1026,7 +1256,7 @@ impl Machine { } } else { self.unmapped += 1; - self.abort(address, false, false, !load); + self.abort_with_status(address, exception::status::SYNC_EXTERNAL, false, !load); return Ok(()); }; if let Some(value) = read { @@ -1273,6 +1503,131 @@ mod tests { Exit::Psci { .. } )); } + + #[test] + fn brk_takes_an_el1_breakpoint_exception_with_its_immediate_in_the_syndrome() { + let mut machine = machine(); + machine.cpu.system.el = 1; + machine.cpu.system.spsel = true; + machine.cpu.system.vbar_el1 = 0x40000400; + // `brk #0x800`, the immediate the kernel's `BUG()` uses. + machine + .memory + .write(0x40000000, &0xd4210000u32.to_le_bytes()) + .unwrap(); + // The vector for a synchronous exception from EL1 on SP_EL1: `VBAR + 0x200`. + machine + .memory + .write(0x40000600, &HVC.to_le_bytes()) + .unwrap(); + machine.cpu.pc = 0x40000000; + let mut serial = Pl011::new(|_| {}); + let mut gic = Gicv2::default(); + assert!(matches!( + machine.run(&mut serial, &mut gic).unwrap(), + Exit::Psci { .. } + )); + let esr = machine.cpu.system.esr_el1; + assert_eq!(esr >> 26, 0x3c, "EC is a breakpoint"); + assert_eq!((esr >> 25) & 1, 1, "IL: a 32-bit instruction"); + assert_eq!(esr & 0xffff, 0x800, "ISS is the immediate"); + assert_eq!( + machine.cpu.system.elr_el1, 0x40000000, + "ELR is the BRK itself" + ); + } + + #[test] + fn el2_with_e2h_sends_el1_register_names_to_the_el2_banks() { + let mut machine = machine(); + machine.cpu.system.el = 2; + machine.cpu.system.spsel = true; + machine.cpu.system.hcr_el2 = crate::aarch64::cpu::HCR_E2H; + machine.cpu.system.vbar_el1 = 0x1111; + machine.cpu.system.vbar_el2 = 0x2222; + // movz x0, #0x1234; msr vbar_el1, x0; mrs x1, vbar_el1 + run_to_hvc(&mut machine, &[0xd2824680, 0xd518c000, 0xd538c001, HVC]); + assert_eq!(machine.cpu.x[1], 0x1234); + assert_eq!(machine.cpu.system.vbar_el2, 0x1234); + assert_eq!(machine.cpu.system.vbar_el1, 0x1111); + } + + #[test] + fn el2_without_e2h_writes_the_el1_bank_and_leaves_el2_alone() { + let mut machine = machine(); + machine.cpu.system.el = 2; + machine.cpu.system.spsel = true; + machine.cpu.system.hcr_el2 = 0; + machine.cpu.system.vbar_el1 = 0x1111; + machine.cpu.system.vbar_el2 = 0x2222; + run_to_hvc(&mut machine, &[0xd2824680, 0xd518c000, 0xd538c001, HVC]); + assert_eq!(machine.cpu.x[1], 0x1234); + assert_eq!(machine.cpu.system.vbar_el1, 0x1234); + assert_eq!(machine.cpu.system.vbar_el2, 0x2222); + } + + #[test] + fn step_trace_reports_each_instruction_and_the_system_register_it_changed() { + use parking_lot::Mutex; + use std::sync::Arc; + let mut machine = machine(); + machine.cpu.system.el = 1; + machine.cpu.system.spsel = true; + let steps = Arc::new(Mutex::new(Vec::new())); + let sink = steps.clone(); + machine.set_step_trace(Some(Box::new(move |step| { + sink.lock().push(step.clone()); + }))); + // movz x0, #1; msr vbar_el1, x0; hvc + run_to_hvc(&mut machine, &[0xd2800020, 0xd518c000, HVC]); + let steps = steps.lock(); + let pcs: Vec = steps.iter().map(|s| s.pc).collect(); + assert_eq!(pcs, [0x40000000, 0x40000004, 0x40000008]); + assert!(steps[0].changes.is_empty()); + assert_eq!(steps[1].changes, [("vbar_el1", 0, 1)]); + assert_eq!(steps[1].next_pc, 0x40000008); + } + #[test] + fn memory_trace_reports_each_translated_access_with_its_pc_and_stored_value() { + use parking_lot::Mutex; + use std::sync::Arc; + let mut machine = machine(); + machine.cpu.system.el = 1; + machine.cpu.system.spsel = true; + let accesses = Arc::new(Mutex::new(Vec::new())); + let sink = accesses.clone(); + machine.set_memory_trace(Some(Box::new(move |access| sink.lock().push(*access)))); + // movz x0,#0x1234; movz x1,#0x4000,lsl #16; add x1,x1,#0x800; str x0,[x1]; + // ldr w2,[x1,#4] + run_to_hvc( + &mut machine, + &[ + 0xd2824680, 0xd2a80001, 0x91200021, 0xf9000020, 0xb9400422, HVC, + ], + ); + let accesses = accesses.lock(); + assert_eq!( + *accesses, + [ + MemoryAccess { + pc: 0x4000000c, + virtual_address: 0x40000800, + physical: 0x40000800, + width: 8, + write: true, + value: 0x1234, + }, + MemoryAccess { + pc: 0x40000010, + virtual_address: 0x40000804, + physical: 0x40000804, + width: 4, + write: false, + value: 0, + }, + ] + ); + } const LDXR_X1_X0: u32 = 0xc85f7c01; const STXR_W2_X3_X0: u32 = 0xc8027c03; const HVC: u32 = 0xd4000002; diff --git a/src/emulator/src/system/mod.rs b/src/emulator/src/system/mod.rs index 2b4740a..d216a5e 100644 --- a/src/emulator/src/system/mod.rs +++ b/src/emulator/src/system/mod.rs @@ -4,17 +4,23 @@ pub mod afdt; pub mod boot; +pub mod dtb; pub mod esr; pub mod fdt; +pub mod m1; pub mod machine; +pub mod monitor; pub mod psci; pub mod smp; pub mod xnu; -use crate::aarch64::{Cpu, decode, translate}; +use crate::aarch64::{Cpu, host, translate}; use crate::devices::bus::Device; +use crate::devices::pci::PciHost; +use crate::devices::virtio_pci::VirtioBlockPci; use crate::devices::{gicv2::Gicv2, gicv3::Gicv3, pl011::Pl011}; use crate::memory::{GuestMemory, PhysicalMemory, Region, SharedMemory}; +use alloc::boxed::Box; use alloc::sync::Arc; use core::{ sync::atomic::{AtomicBool, Ordering}, @@ -31,6 +37,14 @@ pub const DEFAULT_CMDLINE: &str = "console=ttyAMA0 earlycon=pl011,0x9000000 noka pub const DEFAULT_XNU_CMDLINE: &str = "-v serial=3 debug=0x14e keepsyms=1 serial-device-name=uart0"; /// XNU sizes its zones from RAM; 128 MiB leaves it little to work with. pub const XNU_RAM_SIZE: usize = 1 << 30; +/// RAM a firmware boot gets by default: 4 GiB, which is the RAM the firmware's built-in +/// platform description assumes when no device tree is passed (QEMU's run script uses `-m 4G`). +/// The region is zero-filled lazily by the host, so it costs what the guest touches. +pub const FIRMWARE_RAM_SIZE: usize = 4 << 30; +/// The distributor's `ITLinesNumber` on a firmware board: 288 INTIDs. `arm-gic` sizes a +/// firmware's interrupt table from it, and an interrupt source beyond the table, such as the +/// physical timer's PPI, cannot be registered when it reads zero. +pub const FIRMWARE_IT_LINES: u8 = 8; #[derive(Debug)] pub enum Error { @@ -92,6 +106,8 @@ pub struct BootReport { pub executed_blocks: u64, pub timer_interrupts: u64, pub unmapped_accesses: u64, + /// Accesses to arm-io devices the M1 model does not implement (read as zero). + pub unmodeled_device_accesses: u64, pub stalled_at: Option, pub instruction: Option, pub last_fault_walk: Vec, @@ -169,6 +185,11 @@ impl core::fmt::Display for BootReport { for entry in &self.trace { writeln!(f, " pc {:x} sp {:x}", entry.pc, entry.sp)?; } + writeln!( + f, + "unmodeled device accesses {}", + self.unmodeled_device_accesses + )?; writeln!( f, "exits {}, blocks compiled {}, blocks executed {}, timer interrupts {}, unmapped accesses {}, elapsed {:.3}s", @@ -189,6 +210,18 @@ pub struct SystemConfig> { pub ram_size: usize, pub initrd: Option>, pub disk: Option, + pub board: Board, +} + +/// The machine a boot builds around the kernel. +#[derive(Debug, Clone, Default)] +pub enum Board { + /// QEMU `virt`-style board: a GICv2 for Linux, a GICv3 for XNU, a PL011 console. + #[default] + Virt, + /// Apple M1 (t8103) SoC ([`m1`]): an AIC and an S5L console. Linux only. The + /// caller supplies the device tree, such as the Asahi kernel's `t8103-j274.dtb`. + AppleM1 { device_tree: Vec }, } impl Default for SystemConfig { @@ -200,6 +233,7 @@ impl Default for SystemConfig { ram_size: RAM_SIZE, initrd: None, disk: None, + board: Board::Virt, } } } @@ -207,10 +241,12 @@ impl Default for SystemConfig { /// The interrupt controller a boot configures: the GICv2 a Linux device tree /// describes, or the GICv3 XNU's platform expects. Shared peripherals (UART, virtio) /// are wired to the GICv2 only; the GICv3 carries the virtual timer, which the -/// machine evaluates live from the timer registers. +/// machine evaluates live from the timer registers. The M1 board has no GIC: its +/// AIC sits in [`m1::Soc`] and presents the virtual timer as a FIQ. enum Intc { V2(Gicv2), V3(Gicv3), + Apple, } impl Intc { fn raise(&mut self, intid: u32) { @@ -231,35 +267,79 @@ impl Intc { fn signalled(&self, cpu: u32) -> bool { match self { Self::V2(gic) => gic.signalled(cpu), - Self::V3(_) => false, + Self::V3(_) | Self::Apple => false, } } /// See [`machine::Machine::vtimer_fiq`]. fn vtimer_fiq(&self) -> Option { match self { - Self::V2(_) => None, + Self::V2(_) | Self::Apple => None, Self::V3(gic) => gic.vtimer_fiq_priority(), } } + /// Whether the CPU sees a FIQ from the virtual timer, which is due at `due`. The + /// M1 AIC presents the timer's own condition with no further gating. + fn timer_fiq(&self, due: bool) -> bool { + matches!(self, Self::Apple) && due + } +} + +/// Peripherals the firmware discovers in its device tree, separate from kernel boots. +fn firmware_devices(ram_size: u64) -> crate::devices::Bus { + use crate::devices::{Bus, BusDevice, fw_cfg, nor_flash, pl031}; + let mut bus = Bus::new(); + bus.attach(BusDevice::new( + fw_cfg::BASE, + fw_cfg::LEN, + fw_cfg::FwCfg::new(ram_size), + )); + bus.attach(BusDevice::new( + nor_flash::BASE, + nor_flash::LEN, + nor_flash::NorFlash::new(), + )); + let seconds = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as u32; + bus.attach(BusDevice::new( + pl031::BASE, + pl031::LEN, + pl031::Pl031::new(seconds), + )); + bus } +/// The bus of one board. The fixed windows (QEMU `virt`'s UART, GIC and virtio) exist +/// only on the virt board; on the M1 board `soc` answers instead, and the GIC accessors +/// below are never reached with [`Intc::Apple`]. struct SystemDevices<'a, V> { - serial: &'a mut Pl011, + serial: Option<&'a mut Pl011>, gic: &'a mut Intc, virtio: Option<&'a mut V>, + soc: Option<&'a mut m1::Soc>, + pci: Option<&'a mut PciHost>, + firmware: Option<&'a mut crate::devices::Bus>, } impl machine::DeviceIo for SystemDevices<'_, V> { fn read_uart(&mut self, offset: u64, size: u8) -> u64 { - self.serial.read(offset, size) + self.serial + .as_mut() + .expect("the fixed UART window exists only on the virt board, which has a PL011") + .read(offset, size) } fn write_uart(&mut self, offset: u64, size: u8, value: u64) { - self.serial.write(offset, size, value); + self.serial + .as_mut() + .expect("the fixed UART window exists only on the virt board, which has a PL011") + .write(offset, size, value); } fn read_gic_distributor(&mut self, cpu_id: u32, offset: u64, size: u8) -> u64 { match self.gic { Intc::V2(gic) => gic.read_distributor_for(cpu_id, offset, size), Intc::V3(gic) => gic.read_distributor(offset, size), + Intc::Apple => 0, } } fn write_gic_distributor(&mut self, cpu_id: u32, offset: u64, size: u8, value: u64) { @@ -268,13 +348,14 @@ impl machine::DeviceIo for SystemDevices<'_, V> { gic.write_distributor_for(cpu_id, offset, size, value); } Intc::V3(gic) => gic.write_distributor(offset, size, value), + Intc::Apple => {} } } fn read_gic_cpu(&mut self, cpu_id: u32, offset: u64, size: u8) -> u64 { match self.gic { Intc::V2(gic) => gic.read_cpu_for(cpu_id, offset, size), // The GICv3 CPU interface is the `ICC_*` system registers. - Intc::V3(_) => 0, + Intc::V3(_) | Intc::Apple => 0, } } fn write_gic_cpu(&mut self, cpu_id: u32, offset: u64, size: u8, value: u64) { @@ -285,7 +366,7 @@ impl machine::DeviceIo for SystemDevices<'_, V> { fn read_gic_redistributor(&mut self, offset: u64, size: u8) -> u64 { match self.gic { Intc::V3(gic) => gic.read_redistributor(offset, size), - Intc::V2(_) => 0, + Intc::V2(_) | Intc::Apple => 0, } } fn write_gic_redistributor(&mut self, offset: u64, size: u8, value: u64) { @@ -305,8 +386,34 @@ impl machine::DeviceIo for SystemDevices<'_, V> { v.write(offset, size, value); } } + fn read_board(&mut self, cpu_id: u32, address: u64, size: u8) -> Option { + if let Some(soc) = self.soc.as_mut() { + soc.read(cpu_id, address, size) + } else { + self.firmware.as_mut()?.read(address, size) + } + } + fn write_board(&mut self, cpu_id: u32, address: u64, size: u8, value: u64) -> bool { + if let Some(soc) = self.soc.as_mut() { + soc.write(cpu_id, address, size, value) + } else { + self.firmware + .as_mut() + .is_some_and(|bus| bus.write(address, size, value)) + } + } + fn read_pci(&mut self, address: u64, size: u8) -> Option { + self.pci.as_mut()?.read(address, size) + } + fn write_pci(&mut self, address: u64, size: u8, value: u64) -> bool { + self.pci + .as_mut() + .is_some_and(|pci| pci.write(address, size, value)) + } + fn has_fixed_windows(&self) -> bool { + self.soc.is_none() + } } - /// Boot a raw Linux Image with live serial output on stdout and a bounded wall-clock run. /// Execution faults are returned in the report with the full guest state, not discarded. pub fn boot(image: &[u8], timeout: Duration, cmdline: &str) -> Result { @@ -357,15 +464,56 @@ pub fn boot_with_options( ram_size: RAM_SIZE, initrd: None, disk: None, + board: Board::Virt, }, sink, ) } /// Boot with full system configuration including optional initrd and configurable RAM size. -pub fn boot_system( +pub fn boot_system( image: &[u8], config: SystemConfig, + sink: impl FnMut(u8) + Send + 'static, +) -> Result { + boot_machine(Payload::Kernel(image), config, sink) +} + +/// Boot a firmware image the way QEMU's `-bios` does: the image is mapped read-only at +/// [`boot::FIRMWARE_BASE`], the core resets there, and no kernel is loaded. The machine +/// is the `virt` board with a GICv3, as tinted-boot is run. As under QEMU's `-bios`, `x0` +/// is zero and a device tree sits at RAM base. A disk is a virtio-blk function on PCI; there is no initrd. +pub fn boot_firmware( + firmware: &[u8], + config: SystemConfig, + sink: impl FnMut(u8) + Send + 'static, +) -> Result { + boot_machine(Payload::Firmware(firmware), config, sink) +} + +/// What a boot starts: a kernel loaded into RAM, or firmware run from flash. +#[derive(Clone, Copy)] +enum Payload<'a> { + Kernel(&'a [u8]), + Firmware(&'a [u8]), +} + +/// The root bus a firmware boot sees: the host bridge at 00:00.0, and the disk, if any, as a +/// virtio-blk function in slot 1. +fn firmware_pci( + disk: Option, + memory: SharedMemory, +) -> PciHost { + let mut host = PciHost::new(); + if let Some(backend) = disk { + host.attach(1, Box::new(VirtioBlockPci::new(backend, memory))); + } + host +} + +fn boot_machine( + payload: Payload<'_>, + config: SystemConfig, mut sink: impl FnMut(u8) + Send + 'static, ) -> Result { let ram_size = if config.ram_size > 0 { @@ -375,26 +523,69 @@ pub fn boot_system( }; let timeout = config.timeout; let idle_mode = config.idle_mode; - let region = Region::ram(RAM_BASE, vec![0; ram_size]).map_err(Error::Memory)?; - let memory = PhysicalMemory::new(vec![region]).map_err(Error::Memory)?; + let apple = matches!(config.board, Board::AppleM1 { .. }); + let ram_base = if apple { m1::RAM_BASE } else { RAM_BASE }; + let is_xnu = matches!(payload, Payload::Kernel(image) if xnu::is_macho(image)); + let mut regions = vec![Region::ram(ram_base, vec![0; ram_size]).map_err(Error::Memory)?]; + if let Payload::Firmware(flash) = payload { + regions.push(Region::rom(boot::FIRMWARE_BASE, flash.to_vec()).map_err(Error::Memory)?); + } + let memory = PhysicalMemory::new(regions).map_err(Error::Memory)?; let mut shared_mem = SharedMemory::new(memory); let has_virtio = config.disk.is_some(); - let is_xnu = xnu::is_macho(image); - let layout = if is_xnu { - if config.initrd.is_some() || has_virtio { + let layout = match (payload, config.board) { + (Payload::Firmware(_), Board::AppleM1 { .. }) => { return Err(Error::Boot(boot::Error::Unsupported( - "XNU boot has no initrd or virtio disk support", + "the Apple M1 SoC boots no firmware image yet", ))); } - xnu::prepare_boot( - &mut shared_mem, - image, - &config.cmdline, - RAM_BASE, - ram_size as u64, - ) - } else { - boot::prepare_boot( + (Payload::Firmware(_), Board::Virt) => { + if config.initrd.is_some() { + return Err(Error::Boot(boot::Error::Unsupported( + "firmware boot has no initrd; the firmware reads its files from a disk", + ))); + } + boot::prepare_firmware(&mut shared_mem, ram_base, ram_size as u64) + } + (Payload::Kernel(image), Board::AppleM1 { device_tree }) if is_xnu => { + xnu::prepare_boot_with_tree( + &mut shared_mem, + image, + &config.cmdline, + &device_tree, + ram_base, + ram_size as u64, + ) + } + (Payload::Kernel(image), Board::AppleM1 { device_tree }) => { + if has_virtio || config.initrd.is_some() { + return Err(Error::Boot(boot::Error::Unsupported( + "the Apple M1 SoC has no initrd or virtio disk support yet", + ))); + } + m1::prepare( + &mut shared_mem, + image, + &device_tree, + &config.cmdline, + ram_size as u64, + ) + } + (Payload::Kernel(image), Board::Virt) if is_xnu => { + if config.initrd.is_some() || has_virtio { + return Err(Error::Boot(boot::Error::Unsupported( + "XNU boot has no initrd or virtio disk support", + ))); + } + xnu::prepare_boot( + &mut shared_mem, + image, + &config.cmdline, + RAM_BASE, + ram_size as u64, + ) + } + (Payload::Kernel(image), Board::Virt) => boot::prepare_boot( &mut shared_mem, image, config.initrd.as_deref(), @@ -403,33 +594,63 @@ pub fn boot_system( ram_size as u64, 1, has_virtio, - ) + ), } .map_err(Error::Boot)?; let console = Arc::new(Mutex::new(Vec::new())); let serial_console = console.clone(); - let uart_line = Arc::new(AtomicBool::new(false)); - let serial_line = uart_line.clone(); - let mut serial = Pl011::new(move |byte| { + let emit = move |byte: u8| { serial_console.lock().push(byte); sink(byte); - }) - .with_line(move |level| serial_line.store(level, Ordering::Release)); - let mut gic = if is_xnu { - Intc::V3(Gicv3::default()) + }; + let uart_line = Arc::new(AtomicBool::new(false)); + let serial_line = uart_line.clone(); + let (mut serial, mut soc, mut gic) = if apple { + (None, Some(m1::Soc::new(emit)), Intc::Apple) } else { - Intc::V2(Gicv2::default()) + let serial = + Pl011::new(emit).with_line(move |level| serial_line.store(level, Ordering::Release)); + let gic = match payload { + Payload::Firmware(_) => Intc::V3(Gicv3::default().with_it_lines(FIRMWARE_IT_LINES)), + Payload::Kernel(_) if is_xnu => Intc::V3(Gicv3::default()), + Payload::Kernel(_) => Intc::V2(Gicv2::default()), + }; + (Some(serial), None, gic) }; let virtio_line = Arc::new(AtomicBool::new(false)); let virtio_notify = virtio_line.clone(); - let mut virtio = config.disk.map(|backend| { - crate::devices::VirtioBlock::new(backend, shared_mem.clone()) - .with_line(move |level| virtio_notify.store(level, Ordering::Release)) - }); + // A firmware boot reaches its disk over PCI, as under QEMU's `-bios`. A kernel boot on the + // virt board keeps the MMIO transport the Linux device tree describes. + let (mut virtio, mut pci) = match (payload, config.disk) { + (Payload::Kernel(_), disk) => ( + disk.map(|backend| { + crate::devices::VirtioBlock::new(backend, shared_mem.clone()) + .with_line(move |level| virtio_notify.store(level, Ordering::Release)) + }), + None, + ), + (Payload::Firmware(_), disk) => (None, Some(firmware_pci(disk, shared_mem.clone()))), + }; + let mut firmware = + matches!(payload, Payload::Firmware(_)).then(|| firmware_devices(ram_size as u64)); let mut machine = machine::Machine::new(shared_mem); + if apple { + m1::identify(&mut machine.cpu); + } + if matches!(payload, Payload::Firmware(_)) { + // The firmware drives a GICv3 through `ICC_*` system registers, and checks the GIC + // field of ID_AA64PFR0_EL1 before it uses them, as QEMU reports for gic-version=3. + machine.cpu.system.id_aa64pfr0_el1 |= 1 << 24; + } machine.cpu.pc = layout.entry; machine.cpu.x[0] = layout.boot_info; machine.cpu.x[1..4].fill(0); + if matches!(payload, Payload::Kernel(image) if is_xnu && xnu::is_fileset(image)) { + // A kernel collection enters at its reset trampoline: x0 is the reset type (0 for + // a cold boot) and x1 the boot arguments. + machine.cpu.x[0] = 0; + machine.cpu.x[1] = layout.boot_info; + } let started = Instant::now(); let mut exits = 0; let mut timer_interrupts = 0; @@ -437,22 +658,34 @@ pub fn boot_system( if started.elapsed() >= timeout { break StopReason::Timeout; } - if uart_line.load(Ordering::Acquire) { - gic.raise(fdt::UART_INTID); - } else { - gic.lower(fdt::UART_INTID); - } - if virtio_line.load(Ordering::Acquire) { - gic.raise(fdt::VIRTIO_INTID); - } else { - gic.lower(fdt::VIRTIO_INTID); + match soc.as_mut() { + Some(soc) => soc.sync_lines(), + None => { + if uart_line.load(Ordering::Acquire) { + gic.raise(fdt::UART_INTID); + } else { + gic.lower(fdt::UART_INTID); + } + if virtio_line.load(Ordering::Acquire) { + gic.raise(fdt::VIRTIO_INTID); + } else { + gic.lower(fdt::VIRTIO_INTID); + } + } } - machine.irq_line = gic.signalled(0); + machine.irq_line = match &soc { + Some(soc) => soc.irq_pending(0), + None => gic.signalled(0), + }; + machine.fiq_line = gic.timer_fiq(machine.timer_due()); machine.vtimer_fiq = gic.vtimer_fiq(); let mut dev = SystemDevices { - serial: &mut serial, + serial: serial.as_mut(), gic: &mut gic, virtio: virtio.as_mut(), + soc: soc.as_mut(), + pci: pci.as_mut(), + firmware: firmware.as_mut(), }; match machine.run_with_devices(&mut dev) { Ok(machine::Exit::Timer) => { @@ -522,7 +755,7 @@ pub fn boot_system( if !is_xnu && machine.cpu.system.sctlr_el1 & 1 == 0 && sp != 0 - && !(RAM_BASE..=RAM_BASE + ram_size as u64).contains(&sp) + && !(ram_base..=ram_base + ram_size as u64).contains(&sp) { break StopReason::LostStack; } @@ -544,6 +777,7 @@ pub fn boot_system( executed_blocks: machine.blocks_run, timer_interrupts, unmapped_accesses: machine.unmapped, + unmodeled_device_accesses: soc.as_ref().map_or(0, |soc| soc.unmodeled_accesses()), stalled_at, instruction, last_fault_walk, @@ -571,55 +805,74 @@ fn instruction_diagnostic( let mut bytes = [0; 4]; machine.memory.read(physical, &mut bytes).ok()?; let word = u32::from_le_bytes(bytes); - let decoded = decode::decode(word); + let decoded = volt_isa_aarch64::decode::decode(word); last = Some(InstructionDiagnostic { virtual_address: address, physical_address: physical, word, decoded: decoded.is_ok(), }); - if !stalled || decoded.is_err() || decoded.is_ok_and(|instruction| instruction.terminates()) + if !stalled + || decoded.is_err() + || decoded.is_ok_and(|instruction| host::ends_block(&instruction, false)) { break; } } last } +/// The descriptors a translation of `address` reads, top level first, for the granule +/// and VA size the TCR selects. It follows `translate::walk` step for step, so the +/// report shows what the translation saw, for any granule. fn walk_diagnostic(cpu: &Cpu, memory: &impl GuestMemory, address: u64) -> Vec { let mut entries = Vec::new(); if address == 0 || cpu.system.sctlr_el1 & 1 == 0 { return entries; } - let upper = address >> 63 != 0; - let size = if upper { - (cpu.system.tcr_el1 >> 16) & 63 + let tcr = cpu.system.tcr_el1; + let (granule, va_bits, root) = if address >> 63 != 0 { + let granule = [0u32, 14, 12, 16][((tcr >> 30) & 3) as usize]; + let size = ((tcr >> 16) & 63) as u32; + (granule, 64 - size, cpu.system.ttbr1_el1) } else { - cpu.system.tcr_el1 & 63 + let granule = [12u32, 16, 14, 0][((tcr >> 14) & 3) as usize]; + let size = (tcr & 63) as u32; + (granule, 64 - size, cpu.system.ttbr0_el1) }; - let granule = if upper { - (cpu.system.tcr_el1 >> 30) & 3 - } else { - (cpu.system.tcr_el1 >> 14) & 3 - }; - if (!upper && granule != 0) || (upper && granule != 2) { + if granule == 0 || va_bits > 48 || va_bits <= granule { + return entries; + } + let stride = granule - 3; + let levels = (va_bits - granule).div_ceil(stride); + if levels > 4 { return entries; } - let bits = 64u64.saturating_sub(size); - let levels = bits.saturating_sub(12).div_ceil(9).clamp(1, 4) as u8; - let mut table = if upper { - cpu.system.ttbr1_el1 + let top_bits = va_bits - (granule + stride * (levels - 1)); + let root_alignment = if granule == 12 { + granule } else { - cpu.system.ttbr0_el1 - } & 0x0000_ffff_ffff_f000; - for level in 4 - levels..4 { - let index = (address >> (39 - 9 * level)) & 0x1ff; + (top_bits + 3).max(6) + }; + let address_mask = 0x0000_ffff_ffff_ffff & !((1u64 << granule) - 1); + let mut table = root & 0x0000_ffff_ffff_ffff & !((1u64 << root_alignment) - 1); + for depth in 0..levels { + // `translate::walk` counts levels from the leaf; the report counts from the top. + let from_leaf = levels - 1 - depth; + let shift = granule + stride * from_leaf; + let index_bits = if granule != 12 && depth == 0 { + top_bits + } else { + stride + }; + let index = (address >> shift) & ((1u64 << index_bits) - 1); + let at = table.wrapping_add(index * 8); let mut bytes = [0; 8]; let descriptor = memory - .read(table + index * 8, &mut bytes) + .read(at, &mut bytes) .ok() .map(|_| u64::from_le_bytes(bytes)); entries.push(WalkEntry { - level, + level: (4 - levels + depth) as u8, table, index, descriptor, @@ -627,10 +880,10 @@ fn walk_diagnostic(cpu: &Cpu, memory: &impl GuestMemory, address: u64) -> Vec= 3); } + /// A single-precision `fnmadd` is a fused operation, not an `fmov`. Its `Rm` has + /// bits 18:17 set, which the old FMOV pattern accepted because it left bit 24 unchecked. + #[test] + fn fused_fp_operations_are_not_misread_as_fmov() { + let code = [ + 0x1e2e_1000u32, // fmov s0, #1.0 + 0x1e20_1006, // fmov s6, #2.0 + 0x1f26_0002, // fnmadd s2, s0, s6, s0 => -(1) - (1*2) = -3.0 = 0xc0400000 + 0x1e26_0040, // fmov w0, s2 + 0x5318_7c00, // lsr w0, w0, #24 + 0xd2a1_2001, // movz x1, #0x900, lsl #16 (PL011) + 0x3900_0020, // strb w0, [x1] + 0xd2b0_8000, // movz x0, #0x8400, lsl #16 + 0xf280_0100, // movk x0, #8 (PSCI_SYSTEM_OFF) + 0xd400_0002, // hvc #0 + ]; + let mut image = vec![0; 64]; + image[..4].copy_from_slice(&0x14000010u32.to_le_bytes()); + image[4..8].copy_from_slice(&0xd503201fu32.to_le_bytes()); + image[24..32].copy_from_slice(&2u64.to_le_bytes()); + image[56..60].copy_from_slice(&boot::MAGIC.to_le_bytes()); + for w in code { + image.extend_from_slice(&w.to_le_bytes()); + } + let length = image.len() as u64; + image[16..24].copy_from_slice(&length.to_le_bytes()); + let report = + boot_with_serial(&image, Duration::from_secs(10), DEFAULT_CMDLINE, |_| {}).unwrap(); + assert_eq!(report.reason, StopReason::Shutdown, "{report}"); + assert_eq!(report.console, b"\xc0", "high byte of fnmadd s2 result"); + } + fn create_test_image(instructions: &[u32]) -> Vec { let mut image = vec![0; 64]; image[..4].copy_from_slice(&0x14000010u32.to_le_bytes()); @@ -771,6 +1056,7 @@ mod tests { ram_size: 256 << 20, initrd: Some(b"test initrd".to_vec()), disk: None, + board: Board::Virt, }, |_| {}, ) @@ -799,6 +1085,7 @@ mod tests { ram_size: 128 << 20, initrd: None, disk: Some(disk), + board: Board::Virt, }, |_| {}, ) @@ -806,4 +1093,94 @@ mod tests { assert_eq!(report.reason, StopReason::Shutdown); assert_eq!(report.console, b"C"); } + + /// A raw little-endian instruction stream, as a firmware image is laid out in flash. + fn flash_image(instructions: &[u32]) -> Vec { + instructions.iter().flat_map(|i| i.to_le_bytes()).collect() + } + + #[test] + fn firmware_runs_from_flash_with_x0_zero_as_under_qemu_bios() { + // mov x19, x0 keeps the entry value of x0; then 'F' to the PL011, then PSCI off. + let flash = flash_image(&[ + 0xaa0003f3, // mov x19, x0 + 0xd2a12001, // movz x1, #0x900, lsl #16 (PL011) + 0x528008c2, // movz w2, #70 ('F') + 0x39000022, // strb w2, [x1] + 0xd2b08000, // movz x0, #0x8400, lsl #16 + 0xf2800100, // movk x0, #8 (PSCI_SYSTEM_OFF) + 0xd4000002, // hvc #0 + ]); + let report = boot_firmware( + &flash, + SystemConfig::> { + timeout: Duration::from_secs(5), + cmdline: String::new(), + idle_mode: IdleMode::Paced, + ram_size: 256 << 20, + initrd: None, + disk: None, + board: Board::Virt, + }, + |_| {}, + ) + .unwrap(); + assert_eq!(report.reason, StopReason::Shutdown, "{report}"); + assert_eq!(report.console, b"F"); + assert_eq!(report.layout.entry, boot::FIRMWARE_BASE); + assert_eq!(report.layout.boot_info, 0); + assert_eq!(report.cpu.x[19], 0); + assert_eq!(report.unmapped_accesses, 0); + } + + /// A firmware that reads the four bytes of the vendor and device IDs at ECAM slot 1 and + /// writes each to the PL011, then powers off. The ECAM window is 0x40_1000_0000. + fn read_slot_one_ids() -> Vec { + vec![ + 0xd2c00801, // movz x1, #0x40, lsl #32 + 0xf2a20001, // movk x1, #0x1000, lsl #16 + 0xf2900001, // movk x1, #0x8000 (slot 1 = bit 15) + 0xd2a12000, // movz x0, #0x900, lsl #16 (PL011) + 0x39400022, // ldrb w2, [x1] + 0x39000002, // strb w2, [x0] + 0x39400422, // ldrb w2, [x1, #1] + 0x39000002, // strb w2, [x0] + 0x39400822, // ldrb w2, [x1, #2] + 0x39000002, // strb w2, [x0] + 0x39400c22, // ldrb w2, [x1, #3] + 0x39000002, // strb w2, [x0] + 0xd2b08000, // movz x0, #0x8400, lsl #16 + 0xf2800100, // movk x0, #8 (PSCI_SYSTEM_OFF) + 0xd4000002, // hvc #0 + ] + } + + fn firmware_config(disk: Option>) -> SystemConfig> { + SystemConfig { + timeout: Duration::from_secs(5), + cmdline: String::new(), + idle_mode: IdleMode::Paced, + ram_size: 256 << 20, + initrd: None, + disk, + board: Board::Virt, + } + } + + #[test] + fn firmware_finds_its_disk_as_a_virtio_function_in_slot_one() { + let flash = flash_image(&read_slot_one_ids()); + let report = boot_firmware(&flash, firmware_config(Some(vec![0; 512])), |_| {}).unwrap(); + assert_eq!(report.reason, StopReason::Shutdown, "{report}"); + // Vendor 0x1af4 and device 0x1042, little-endian. + assert_eq!(report.console, [0xf4, 0x1a, 0x42, 0x10]); + } + + #[test] + fn firmware_without_a_disk_finds_slot_one_empty() { + let flash = flash_image(&read_slot_one_ids()); + let report = boot_firmware(&flash, firmware_config(None), |_| {}).unwrap(); + assert_eq!(report.reason, StopReason::Shutdown, "{report}"); + assert_eq!(report.console, [0xff; 4]); + } } diff --git a/src/emulator/src/system/monitor.rs b/src/emulator/src/system/monitor.rs new file mode 100644 index 0000000..ee1fa82 --- /dev/null +++ b/src/emulator/src/system/monitor.rs @@ -0,0 +1,745 @@ +//! Apple's GL2 monitor (SPTM), entered at EL2 with the MMU off. +//! +//! The boot firmware copies the monitor image into DRAM and jumps to its entry at the +//! image's *physical* address, with x0 the physical address of a boot structure. The +//! monitor builds its own translation tables (its `__BOOTDATA` and `__LATE_CONST` +//! segments are zero in the file), sets `HCR_EL2.{E2H,TGE}`, and turns the MMU on +//! itself. [`load`] does the same. The structure's fields come from the entry code and +//! from where the monitor reads its copy of it: +//! +//! | offset | value | +//! |--------|------------------------------------------------------------------------| +//! | 0 | count (the code tests `>= 3`) | +//! | 8 | virtual address of the first DRAM byte in the monitor's map (`x22`) | +//! | 16 | physical address of the first DRAM byte (`x23`) | +//! | 24 | bytes of DRAM to map (`x24`) | +//! | 0x60 | device tree address, as the monitor sees it (the linear map) | +//! | 0x68 | device tree size in bytes (32 bits) | +//! | 1152 | physical address of a scratch buffer, zeroed and stamped `"SPTM"` | +//! +//! The monitor maps all of DRAM linearly (`virtual = physical + [8] - [16]`) in 32 MiB +//! blocks, which drop the low 25 bits of both addresses, so both bases are block-aligned. +//! Its own image sits inside that map at its link address, which fixes `[8]`: the image's +//! physical base equals its virtual base modulo 32 MiB. Everything the monitor reads +//! before it has its own mappings (the boot structure, the device tree, the scratch +//! buffer, the kernel collection it protects) is reached through that map, so it all lies +//! in DRAM. Seen with the per-instruction and memory traces (`Machine::set_step_trace`, +//! `Machine::set_memory_trace`). + +use crate::{ + memory::{GuestMemory, MemoryError}, + system::xnu, +}; + +/// The monitor's block-mapping granularity: 32 MiB (a 16 KiB-granule level-2 block). +pub const BLOCK: u64 = 32 << 20; +/// Offsets of the device-tree address and size in the boot structure, read from the +/// monitor's copy of it (`ldr x9, [x8]; ldr w8, [x8, #8]` at `+0x60`). +const DEVICE_TREE_SLOT: u64 = 0x60; +/// Offset of the scratch-buffer pointer in the boot structure (`ldr x5, [x20, #1152]`). +const SCRATCH_SLOT: u64 = 1152; +/// Bytes the entry code zeroes at the scratch buffer (`mov x7, #0x17c0`). +const SCRATCH_BYTES: u64 = 0x17c0; +/// Bytes of boot structure written: through the scratch slot. +const BOOT_BYTES: usize = SCRATCH_SLOT as usize + 8; + +#[derive(Debug)] +pub enum Error { + /// The image is not a Mach-O executable the loader can place. + Image(crate::system::boot::Error), + Memory(MemoryError), + /// The physical image base is not congruent with the virtual base modulo 32 MiB. + Misaligned { + virtual_base: u64, + physical_base: u64, + }, + /// DRAM is not 32 MiB-aligned, or is smaller than one block. A size under one block + /// makes the monitor's mapping loop wrap and run for ever. + Dram { + base: u64, + size: u64, + }, + /// A part the monitor reads through its linear map is outside DRAM. + OutsideDram { + what: &'static str, + physical: u64, + }, + /// The device tree is not a well-formed Apple flattened device tree. + DeviceTree(&'static str), + /// The entry point lies in no part of the image. + NoEntryPart, +} + +impl From for Error { + fn from(error: MemoryError) -> Self { + Self::Memory(error) + } +} + +/// Where the firmware puts each part of the monitor, all physical addresses. +#[derive(Clone, Copy, Debug)] +pub struct Layout { + /// First byte of DRAM and its size, both multiples of [`BLOCK`]. + pub dram_base: u64, + pub dram_size: u64, + /// Physical address of the image's lowest segment (`__TEXT`). Must be congruent + /// with that segment's link address modulo [`BLOCK`], and inside DRAM. + pub image_phys: u64, + /// Physical address of the boot structure, inside DRAM. + pub boot_phys: u64, + /// Physical address of the monitor's scratch buffer, inside DRAM. + pub scratch_phys: u64, + /// Physical address the device tree is copied to, inside DRAM. + pub device_tree_phys: u64, +} + +/// A monitor image placed in memory, ready to run. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Monitor { + /// Physical address of the entry point (`LC_UNIXTHREAD` pc, relocated). + pub entry: u64, + /// Physical address of the boot structure, passed in x0. + pub boot_phys: u64, +} + +/// Bit 31 of an ADT property length marks a placeholder property; the length is the low 31 +/// bits. The monitor adds the raw length to the property's address, so the flag would make +/// every walk overflow its bounds check. +const PLACEHOLDER: u32 = 1 << 31; + +/// A copy of the Apple flattened device tree `tree` with every placeholder flag cleared. +/// Nodes are `{properties: u32, children: u32}`, each property a 32-byte name, a `u32` +/// length and its value padded to four bytes, then the children. +pub fn without_placeholder_flags(tree: &[u8]) -> Result, Error> { + const TRUNCATED: Error = Error::DeviceTree("device tree is truncated"); + fn word(tree: &[u8], at: usize) -> Result { + let bytes = tree.get(at..at + 4).ok_or(TRUNCATED)?; + Ok(u32::from_le_bytes(bytes.try_into().unwrap())) + } + fn node(tree: &mut [u8], at: usize) -> Result { + let properties = word(tree, at)?; + let children = word(tree, at + 4)?; + let mut at = at + 8; + for _ in 0..properties { + let length = word(tree, at + 32)? & !PLACEHOLDER; + tree.get_mut(at + 32..at + 36) + .ok_or(TRUNCATED)? + .copy_from_slice(&length.to_le_bytes()); + at += 36 + (length as usize).next_multiple_of(4); + if at > tree.len() { + return Err(TRUNCATED); + } + } + for _ in 0..children { + at = node(tree, at)?; + } + Ok(at) + } + let mut copy = tree.to_vec(); + node(&mut copy, 0)?; + Ok(copy) +} + +/// Appends a property `name = value` to the node at `path` of the device tree `tree` +/// (`path` names children of the root, e.g. `["chosen", "memory-map"]`). +pub fn add_property( + tree: &[u8], + path: &[&[u8]], + name: &str, + value: &[u8], +) -> Result, Error> { + const TRUNCATED: Error = Error::DeviceTree("device tree is truncated"); + const NAME: usize = 32; + fn word(tree: &[u8], at: usize) -> Result { + let bytes = tree.get(at..at + 4).ok_or(TRUNCATED)?; + Ok(u32::from_le_bytes(bytes.try_into().unwrap())) + } + /// The offset just past the properties of the node at `at`, and its name. + fn properties(tree: &[u8], at: usize) -> Result<(usize, Option<&[u8]>), Error> { + let mut found = None; + let mut cursor = at + 8; + for _ in 0..word(tree, at)? { + let length = (word(tree, cursor + NAME)? & !PLACEHOLDER) as usize; + let value = cursor + NAME + 4; + if tree.get(cursor..cursor + 5) == Some(b"name\0") { + found = tree.get(value..value + length); + } + cursor = value + length.next_multiple_of(4); + if cursor > tree.len() { + return Err(TRUNCATED); + } + } + Ok((cursor, found.map(|n| n.strip_suffix(b"\0").unwrap_or(n)))) + } + /// The offset just past the node at `at` and everything under it. + fn skip(tree: &[u8], at: usize) -> Result { + let (mut cursor, _) = properties(tree, at)?; + for _ in 0..word(tree, at + 4)? { + cursor = skip(tree, cursor)?; + } + Ok(cursor) + } + /// The offset of the child of the node at `at` named `wanted`. + fn child(tree: &[u8], at: usize, wanted: &[u8]) -> Result { + let (mut cursor, _) = properties(tree, at)?; + for _ in 0..word(tree, at + 4)? { + if properties(tree, cursor)?.1 == Some(wanted) { + return Ok(cursor); + } + cursor = skip(tree, cursor)?; + } + Err(Error::DeviceTree("device tree has no such node")) + } + if name.len() >= NAME || value.len() > u32::MAX as usize { + return Err(Error::DeviceTree("property name or value is too large")); + } + let mut node = 0; + for part in path { + node = child(tree, node, part)?; + } + let (end, _) = properties(tree, node)?; + let mut property = vec![0u8; NAME + 4 + value.len()]; + property[..name.len()].copy_from_slice(name.as_bytes()); + property[NAME..NAME + 4].copy_from_slice(&(value.len() as u32).to_le_bytes()); + property[NAME + 4..].copy_from_slice(value); + property.resize(property.len().next_multiple_of(4), 0); + let mut out = Vec::with_capacity(tree.len() + property.len()); + out.extend_from_slice(&tree[..end]); + out.extend_from_slice(&property); + out.extend_from_slice(&tree[end..]); + let count = word(&out, node)? + 1; + out[node..node + 4].copy_from_slice(&count.to_le_bytes()); + Ok(out) +} + +/// Appends the property `name = {address: u64, size: u64}` to the device tree's +/// `chosen/memory-map` node, the way the boot firmware records a region it loaded. +pub fn add_memory_map_region( + tree: &[u8], + name: &str, + address: u64, + size: u64, +) -> Result, Error> { + let mut value = Vec::with_capacity(16); + value.extend(address.to_le_bytes()); + value.extend(size.to_le_bytes()); + add_property(tree, &[b"chosen", b"memory-map"], name, &value) +} + +/// Places the `MH_EXECUTE` monitor `image` and `device_tree`, and writes the boot structure. +pub fn load( + memory: &mut impl GuestMemory, + image: &[u8], + device_tree: &[u8], + layout: &Layout, +) -> Result { + let kernel = xnu::parse(image).map_err(Error::Image)?; + let device_tree = without_placeholder_flags(device_tree)?; + let base = kernel + .segments + .iter() + .filter(|s| s.vmsize != 0) + .map(|s| s.vmaddr) + .min() + .unwrap_or(0); + let (dram, size) = (layout.dram_base, layout.dram_size); + if dram & (BLOCK - 1) != 0 || size & (BLOCK - 1) != 0 || size < BLOCK { + return Err(Error::Dram { base: dram, size }); + } + if (base ^ layout.image_phys) & (BLOCK - 1) != 0 { + return Err(Error::Misaligned { + virtual_base: base, + physical_base: layout.image_phys, + }); + } + let end = kernel + .segments + .iter() + .map(|s| s.vmaddr + s.vmsize) + .max() + .unwrap_or(base); + for (what, physical, length) in [ + ("image", layout.image_phys, end - base), + ("boot structure", layout.boot_phys, BOOT_BYTES as u64), + ("scratch buffer", layout.scratch_phys, SCRATCH_BYTES), + ( + "device tree", + layout.device_tree_phys, + device_tree.len() as u64, + ), + ] { + if physical < dram || physical + length > dram + size { + return Err(Error::OutsideDram { what, physical }); + } + } + for segment in &kernel.segments { + if segment.vmsize == 0 { + continue; + } + let mut bytes = vec![0u8; segment.vmsize as usize]; + let file = &image[segment.fileoff as usize..][..segment.filesize as usize]; + bytes[..file.len()].copy_from_slice(file); + memory.write(layout.image_phys + (segment.vmaddr - base), &bytes)?; + } + // The image sits at its link address in the linear map, which fixes where DRAM starts. + let dram_va = base.wrapping_sub(layout.image_phys - dram); + let linear = dram_va.wrapping_sub(dram); + let mut boot = [0u8; BOOT_BYTES]; + let mut put = |at: u64, value: u64| { + boot[at as usize..at as usize + 8].copy_from_slice(&value.to_le_bytes()); + }; + put(0, 3); + put(8, dram_va); + put(16, dram); + put(24, size); + put( + DEVICE_TREE_SLOT, + layout.device_tree_phys.wrapping_add(linear), + ); + put(DEVICE_TREE_SLOT + 8, device_tree.len() as u64); + put(SCRATCH_SLOT, layout.scratch_phys); + memory.write(layout.device_tree_phys, &device_tree)?; + memory.write(layout.boot_phys, &boot)?; + memory.write(layout.scratch_phys, &vec![0u8; SCRATCH_BYTES as usize])?; + Ok(Monitor { + entry: layout.image_phys + (kernel.entry - base), + boot_phys: layout.boot_phys, + }) +} + +/// The lowest nonempty segment address of `kernel`: the image's link base. +fn image_base(kernel: &xnu::Kernel) -> u64 { + kernel + .segments + .iter() + .filter(|s| s.vmsize != 0) + .map(|s| s.vmaddr) + .min() + .unwrap_or(0) +} + +/// Places a Mach-O image so that its lowest segment is at physical `phys`. `phys` must +/// match the link base modulo [`BLOCK`], so the monitor's linear map reaches the image at +/// its link address. Returns the physical entry point. +pub fn place_image(memory: &mut impl GuestMemory, image: &[u8], phys: u64) -> Result { + let kernel = xnu::parse(image).map_err(Error::Image)?; + let base = image_base(&kernel); + if (base ^ phys) & (BLOCK - 1) != 0 { + return Err(Error::Misaligned { + virtual_base: base, + physical_base: phys, + }); + } + write_segments(memory, image, &kernel, base, phys)?; + Ok(phys + (kernel.entry - base)) +} + +/// Writes each segment's file bytes, zero-filled to its size, at `phys + (vmaddr - base)`. +fn write_segments( + memory: &mut impl GuestMemory, + image: &[u8], + kernel: &xnu::Kernel, + base: u64, + phys: u64, +) -> Result<(), Error> { + for segment in &kernel.segments { + if segment.vmsize == 0 { + continue; + } + let mut bytes = vec![0u8; segment.vmsize as usize]; + let file = &image[segment.fileoff as usize..][..segment.filesize as usize]; + bytes[..file.len()].copy_from_slice(file); + memory.write(phys + (segment.vmaddr - base), &bytes)?; + } + Ok(()) +} + +/// A named span of a Mach-O image, in the monitor's naming: link addresses `lo..hi`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Part { + pub name: &'static str, + pub lo: u64, + pub hi: u64, +} + +/// The parts of a Mach-O image, as the monitor names them. `ro` covers `__TEXT`, +/// `__PRELINK_TEXT`, `__DATA_CONST` and `__LATE_CONST`; `rx` covers `__TEXT_EXEC` and +/// `__LAST`; `bx` is `__TEXT_BOOT_EXEC`; `rw` covers `__DATA` and `__BOOTDATA`; `le` is +/// `__LINKEDIT`; `rs` is `__DATA_SPTM` where present; `virt` is the whole image; `entry` +/// is the segment holding the entry point. A part is the smallest span covering its +/// segments and is copied as one piece, so the monitor maps it contiguously. +pub fn image_parts(image: &[u8]) -> Result, Error> { + let kernel = xnu::parse(image).map_err(Error::Image)?; + let base = image_base(&kernel); + let end = kernel + .segments + .iter() + .map(|s| s.vmaddr + s.vmsize) + .max() + .unwrap_or(base); + let span = |names: &[&[u8]]| -> Option<(u64, u64)> { + kernel + .segments + .iter() + .filter(|s| s.vmsize != 0 && names.contains(&s.name())) + .map(|s| (s.vmaddr, s.vmaddr + s.vmsize)) + .fold(None, |acc, (lo, hi)| match acc { + None => Some((lo, hi)), + Some((a, b)) => Some((a.min(lo), b.max(hi))), + }) + }; + let entry = kernel + .segments + .iter() + .find(|s| s.vmsize != 0 && (s.vmaddr..s.vmaddr + s.vmsize).contains(&kernel.entry)) + .map(|s| (s.vmaddr, s.vmaddr + s.vmsize)); + let spans = [ + ( + "ro", + span(&[ + b"__TEXT", + b"__PRELINK_TEXT", + b"__DATA_CONST", + b"__LATE_CONST", + ]), + ), + ("rx", span(&[b"__TEXT_EXEC", b"__LAST"])), + ("bx", span(&[b"__TEXT_BOOT_EXEC"])), + ("rw", span(&[b"__DATA", b"__BOOTDATA"])), + ("le", span(&[b"__LINKEDIT"])), + ("rs", span(&[b"__DATA_SPTM"])), + ("virt", Some((base, end))), + ("entry", entry), + ]; + Ok(spans + .into_iter() + .filter_map(|(name, range)| range.map(|(lo, hi)| Part { name, lo, hi })) + .collect()) +} + +/// Copies the link span of `part` from `image` to physical `pa`: the file bytes of each +/// segment inside it, zeros elsewhere. +pub fn write_part( + memory: &mut impl GuestMemory, + image: &[u8], + part: &Part, + pa: u64, +) -> Result<(), Error> { + let kernel = xnu::parse(image).map_err(Error::Image)?; + let mut bytes = vec![0u8; (part.hi - part.lo) as usize]; + for segment in &kernel.segments { + let lo = segment.vmaddr.max(part.lo); + let hi = (segment.vmaddr + segment.filesize).min(part.hi); + if lo >= hi { + continue; + } + let from = segment.fileoff + (lo - segment.vmaddr); + let file = &image[from as usize..][..(hi - lo) as usize]; + bytes[(lo - part.lo) as usize..(hi - part.lo) as usize].copy_from_slice(file); + } + memory.write(pa, &bytes)?; + Ok(()) +} + +/// Writes every part of `image` at the address `place` gives it, and returns the image's +/// `chosen/memory-map` regions as `(prefix-part, address, size)`. `entry` is not written: +/// it lies at the same offset within the part that holds the entry point. +pub fn place_parts( + memory: &mut impl GuestMemory, + image: &[u8], + prefix: &str, + place: impl Fn(&Part) -> u64, +) -> Result, Error> { + let parts = image_parts(image)?; + let mut regions = Vec::new(); + for part in parts.iter().filter(|p| p.name != "entry") { + let pa = place(part); + write_part(memory, image, part, pa)?; + regions.push((format!("{prefix}-{}", part.name), pa, part.hi - part.lo)); + } + if let Some(entry) = parts.iter().find(|p| p.name == "entry") { + let holder = parts + .iter() + .find(|p| !matches!(p.name, "entry" | "virt") && p.lo <= entry.lo && entry.hi <= p.hi) + .ok_or(Error::NoEntryPart)?; + let pa = place(holder) + (entry.lo - holder.lo); + regions.push((format!("{prefix}-entry"), pa, entry.hi - entry.lo)); + } + Ok(regions) +} + +/// Appends every region of `regions` to `chosen/memory-map` of `tree`, in order. +pub fn add_memory_map_regions( + tree: &[u8], + regions: &[(String, u64, u64)], +) -> Result, Error> { + let mut tree = tree.to_vec(); + for (name, address, size) in regions { + tree = add_memory_map_region(&tree, name, *address, *size)?; + } + Ok(tree) +} + +/// Sets the CPU to the state the firmware leaves for the monitor: EL2, MMU off, at the +/// entry's physical address, with the boot structure in x0. +pub fn prepare_cpu(cpu: &mut crate::aarch64::cpu::Cpu, monitor: &Monitor) { + let system = &mut cpu.system; + system.el = 2; + system.spsel = true; + // EL1 is AArch64. The monitor sets `E2H` and `TGE` itself. + system.hcr_el2 = 1 << 31; + system.sctlr_el2 = 0; + system.daif = 15; + cpu.pc = monitor.entry; + cpu.x[0] = monitor.boot_phys; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::memory::{PhysicalMemory, Region}; + + /// A minimal `MH_EXECUTE` arm64 image: one `__TEXT` segment holding `payload`, + /// linked at `vmaddr`, with an `LC_UNIXTHREAD` entry `entry_offset` bytes in. + fn image(vmaddr: u64, entry_offset: u64, payload: &[u8]) -> Vec { + let thread = 8 + 8 + 272; + let segment = 72; + let header = 32; + let file_size = 0x4000u64; + let mut out = Vec::new(); + for word in [ + 0xfeed_facf_u32, + 0x0100_000c, + 0, + 2, + 2, + (segment + thread) as u32, + 0, + 0, + ] { + out.extend(word.to_le_bytes()); + } + out.extend(0x19u32.to_le_bytes()); + out.extend((segment as u32).to_le_bytes()); + let mut name = [0u8; 16]; + name[..6].copy_from_slice(b"__TEXT"); + out.extend(name); + for word in [vmaddr, file_size, 0, file_size] { + out.extend(word.to_le_bytes()); + } + for word in [5u32, 5, 0, 0] { + out.extend(word.to_le_bytes()); + } + out.extend(5u32.to_le_bytes()); + out.extend((thread as u32).to_le_bytes()); + out.extend(6u32.to_le_bytes()); + out.extend(68u32.to_le_bytes()); + let mut state = [0u64; 34]; + state[32] = vmaddr + entry_offset; + for word in state { + out.extend(word.to_le_bytes()); + } + out.resize(header + segment + thread, 0); + out.resize(0x200, 0); + out.extend_from_slice(payload); + out.resize(file_size as usize, 0); + out + } + + const DRAM: u64 = 0x8_0000_0000; + const VMADDR: u64 = 0xffff_fff0_2700_4000; + + fn memory() -> PhysicalMemory { + PhysicalMemory::new(vec![ + Region::ram(DRAM, vec![0; (2 * BLOCK) as usize]).unwrap(), + ]) + .unwrap() + } + + /// Two blocks of DRAM, the image at its link offset in the first, the rest below it. + fn layout() -> Layout { + Layout { + dram_base: DRAM, + dram_size: 2 * BLOCK, + image_phys: DRAM + (VMADDR & (BLOCK - 1)), + boot_phys: DRAM + 0x10_0000, + scratch_phys: DRAM + 0x11_0000, + device_tree_phys: DRAM + 0x12_0000, + } + } + + #[test] + fn the_image_is_placed_and_the_boot_structure_describes_dram() { + let image = image(VMADDR, 0x200, &[0xaa, 0xbb, 0xcc, 0xdd]); + let mut memory = memory(); + let layout = layout(); + let monitor = load(&mut memory, &image, &[0; 8], &layout).unwrap(); + assert_eq!(monitor.entry, layout.image_phys + 0x200); + let mut word = [0u8; 8]; + let mut field = |at: u64| { + memory.read(layout.boot_phys + at, &mut word).unwrap(); + u64::from_le_bytes(word) + }; + // The image is at its link address in the linear map, so DRAM's first byte is + // mapped as many bytes below the link address as the image is above DRAM's start. + let dram_va = VMADDR - (layout.image_phys - DRAM); + assert_eq!(field(0), 3); + assert_eq!(field(8), dram_va); + assert_eq!(field(16), DRAM); + assert_eq!(field(24), 2 * BLOCK); + assert_eq!(field(SCRATCH_SLOT), layout.scratch_phys); + assert_eq!( + field(DEVICE_TREE_SLOT), + layout.device_tree_phys - DRAM + dram_va + ); + assert_eq!(field(DEVICE_TREE_SLOT + 8), 8); + let mut payload = [0u8; 4]; + memory + .read(layout.image_phys + 0x200, &mut payload) + .unwrap(); + assert_eq!(payload, [0xaa, 0xbb, 0xcc, 0xdd]); + assert_eq!( + dram_va & (BLOCK - 1), + 0, + "DRAM's virtual base is block-aligned" + ); + } + + #[test] + fn a_physical_base_that_is_not_congruent_with_the_virtual_base_is_refused() { + let layout = Layout { + image_phys: DRAM + 0x8000, + ..layout() + }; + assert!(matches!( + load(&mut memory(), &image(VMADDR, 0x200, &[]), &[0; 8], &layout), + Err(Error::Misaligned { .. }) + )); + } + + #[test] + fn dram_that_is_unaligned_or_under_one_block_is_refused() { + for (base, size) in [ + (DRAM + 0x1000, 2 * BLOCK), + (DRAM, BLOCK - 0x1000), + (DRAM, 0), + ] { + let layout = Layout { + dram_base: base, + dram_size: size, + ..layout() + }; + assert!(matches!( + load(&mut memory(), &image(VMADDR, 0x200, &[]), &[0; 8], &layout), + Err(Error::Dram { .. }) + )); + } + } + + #[test] + fn a_part_the_monitor_reads_through_its_map_must_lie_in_dram() { + let layout = Layout { + boot_phys: DRAM + 2 * BLOCK, + ..layout() + }; + assert!(matches!( + load(&mut memory(), &image(VMADDR, 0x200, &[]), &[0; 8], &layout), + Err(Error::OutsideDram { + what: "boot structure", + .. + }) + )); + } + + /// A root node with one property `a` of `length` (raw, flags included) holding `data`, + /// and one child with no properties. + fn tree(length: u32, data: &[u8]) -> Vec { + let mut out = Vec::new(); + out.extend(1u32.to_le_bytes()); + out.extend(1u32.to_le_bytes()); + let mut name = [0u8; 32]; + name[0] = b'a'; + out.extend(name); + out.extend(length.to_le_bytes()); + out.extend(data); + out.resize(out.len().next_multiple_of(4), 0); + out.extend([0u8; 8]); + out + } + + #[test] + fn placeholder_flags_are_cleared_from_property_lengths_and_nothing_else_changes() { + let flagged = tree(PLACEHOLDER | 5, b"hello"); + let clean = without_placeholder_flags(&flagged).unwrap(); + assert_eq!(clean, tree(5, b"hello")); + // The child node after the padded value is still found, so the walk stayed in step. + assert_eq!(clean.len(), flagged.len()); + } + + #[test] + fn a_tree_that_ends_inside_a_property_is_refused() { + let mut cut = tree(PLACEHOLDER | 5, b"hello"); + cut.truncate(40); + assert!(matches!( + without_placeholder_flags(&cut), + Err(Error::DeviceTree(_)) + )); + } + + /// A node: its properties (name, value) and children, in the Apple flattened format. + fn adt(properties: &[(&str, &[u8])], children: &[Vec]) -> Vec { + let mut out = Vec::new(); + out.extend((properties.len() as u32).to_le_bytes()); + out.extend((children.len() as u32).to_le_bytes()); + for (name, value) in properties { + let mut field = [0u8; 32]; + field[..name.len()].copy_from_slice(name.as_bytes()); + out.extend(field); + out.extend((value.len() as u32).to_le_bytes()); + out.extend(*value); + out.resize(out.len().next_multiple_of(4), 0); + } + for child in children { + out.extend(child); + } + out + } + + #[test] + fn a_region_lands_in_chosen_memory_map_and_other_nodes_are_untouched() { + let map = |props: &[(&str, &[u8])]| { + let mut all = vec![("name", &b"memory-map\0"[..])]; + all.extend(props); + adt(&all, &[]) + }; + let tree = |inner: Vec| { + adt( + &[("name", b"device-tree\0")], + &[ + adt(&[("name", b"cpus\0")], &[]), + adt(&[("name", b"chosen\0")], &[inner]), + adt(&[("name", b"after\0")], &[]), + ], + ) + }; + let before = tree(map(&[("RAMDisk", &[1; 16])])); + let after = add_memory_map_region(&before, "BootKC-rs", 0x1234, 0x56).unwrap(); + // The same tree, built with the property already present, is byte-identical. + let mut value = Vec::new(); + value.extend(0x1234u64.to_le_bytes()); + value.extend(0x56u64.to_le_bytes()); + assert_eq!( + after, + tree(map(&[("RAMDisk", &[1; 16]), ("BootKC-rs", &value)])) + ); + } + + #[test] + fn a_tree_without_chosen_memory_map_is_refused() { + let tree = adt(&[("name", b"device-tree\0")], &[]); + assert!(matches!( + add_memory_map_region(&tree, "BootKC-rs", 0, 0), + Err(Error::DeviceTree(_)) + )); + } +} diff --git a/src/emulator/src/system/xnu.rs b/src/emulator/src/system/xnu.rs index f1a7115..9a25c88 100644 --- a/src/emulator/src/system/xnu.rs +++ b/src/emulator/src/system/xnu.rs @@ -15,6 +15,9 @@ use crate::memory::GuestMemory; const MH_MAGIC_64: u32 = 0xfeed_facf; const CPU_TYPE_ARM64: u32 = 0x0100_000c; const MH_EXECUTE: u32 = 2; +/// A kernel collection: one top-level header whose load commands list every segment of +/// every component, with `LC_FILESET_ENTRY` naming each component's own Mach-O header. +const MH_FILESET: u32 = 12; const LC_UNIXTHREAD: u32 = 0x5; const LC_SEGMENT_64: u32 = 0x19; const LC_MAIN: u32 = 0x8000_0028; @@ -51,6 +54,16 @@ pub struct Segment { pub vmsize: u64, pub fileoff: u64, pub filesize: u64, + /// `segname`, NUL-padded. + pub name: [u8; 16], +} + +impl Segment { + /// The segment name without its NUL padding (`__TEXT`). + pub fn name(&self) -> &[u8] { + let end = self.name.iter().position(|&b| b == 0).unwrap_or(16); + &self.name[..end] + } } /// What the loader needs from a Mach-O kernel. @@ -67,6 +80,12 @@ pub fn is_macho(image: &[u8]) -> bool { image.len() >= 4 && u32::from_le_bytes(image[..4].try_into().unwrap()) == MH_MAGIC_64 } +/// Whether `image` is a kernel collection (`MH_FILESET`), which enters at its reset +/// trampoline rather than at `_start`. +pub fn is_fileset(image: &[u8]) -> bool { + image.len() >= 16 && u32::from_le_bytes(image[12..16].try_into().unwrap()) == MH_FILESET +} + fn bad(reason: &'static str) -> Error { Error::BadMachO(reason) } @@ -85,8 +104,8 @@ pub fn parse(image: &[u8]) -> Result { if word(4) != CPU_TYPE_ARM64 { return Err(bad("not an arm64 Mach-O")); } - if word(12) != MH_EXECUTE { - return Err(bad("not an executable Mach-O")); + if word(12) != MH_EXECUTE && word(12) != MH_FILESET { + return Err(bad("not an executable or fileset Mach-O")); } let commands_end = HEADER_SIZE .checked_add(word(20) as usize) @@ -114,6 +133,7 @@ pub fn parse(image: &[u8]) -> Result { vmsize: double(at + 32), fileoff: double(at + 40), filesize: double(at + 48), + name: image[at + 8..at + 24].try_into().unwrap(), }; let in_file = segment .fileoff @@ -205,6 +225,20 @@ pub fn prepare_boot( cmdline: &str, ram_base: u64, ram_size: u64, +) -> Result { + let tree = super::afdt::build(ram_base, ram_size, cmdline); + prepare_boot_with_tree(memory, image, cmdline, &tree, ram_base, ram_size) +} + +/// [`prepare_boot`] with the device tree the caller supplies, such as the platform's +/// own ADT for the board. +pub fn prepare_boot_with_tree( + memory: &mut impl GuestMemory, + image: &[u8], + cmdline: &str, + tree: &[u8], + ram_base: u64, + ram_size: u64, ) -> Result { let kernel = parse(image)?; let ram_end = ram_base.checked_add(ram_size).ok_or(Error::NoRoom)?; @@ -235,7 +269,6 @@ pub fn prepare_boot( } let device_tree = align(kernel_end, PAGE)?; - let tree = super::afdt::build(ram_base, ram_size, cmdline); let args_at = align( device_tree .checked_add(tree.len() as u64) @@ -272,7 +305,7 @@ pub fn prepare_boot( .write(physical(segment.vmaddr)?, data) .map_err(Error::Memory)?; } - memory.write(device_tree, &tree).map_err(Error::Memory)?; + memory.write(device_tree, tree).map_err(Error::Memory)?; memory.write(args_at, &args).map_err(Error::Memory)?; Ok(Layout { entry, @@ -443,4 +476,15 @@ mod tests { Err(Error::Unsupported(_)) )); } + + #[test] + fn a_fileset_kernel_parses_like_an_executable_with_the_same_segments() { + let executable = image(Entry::Main(0x800)); + let mut fileset = executable.clone(); + fileset[12..16].copy_from_slice(&MH_FILESET.to_le_bytes()); + assert_eq!(parse(&fileset).unwrap(), parse(&executable).unwrap()); + let mut other = executable; + other[12..16].copy_from_slice(&1u32.to_le_bytes()); + assert!(matches!(parse(&other), Err(Error::BadMachO(_)))); + } } diff --git a/src/emulator/src/user/mod.rs b/src/emulator/src/user/mod.rs index 7dae319..4682c6f 100644 --- a/src/emulator/src/user/mod.rs +++ b/src/emulator/src/user/mod.rs @@ -10,11 +10,12 @@ pub mod syscalls; pub use memory::Space; use crate::aarch64::cpu::{Exclusive, Trap}; -use crate::aarch64::decode::{Instruction, SignExtend, SystemRegister, decode}; +use crate::aarch64::host::run as run_host; use crate::aarch64::{Cache, Cpu}; use crate::memory::{GuestMemory, MemoryError}; use std::ffi::OsString; use std::path::Path; +use volt_isa_aarch64::decode::{Instruction, SignExtend, SystemRegister, decode}; #[derive(Debug)] pub enum Error { @@ -109,7 +110,7 @@ pub fn run(path: &Path, args: &[OsString], env: &[(OsString, OsString)]) -> Resu Trap::SimdStruct => simd_struct(&mut space, &mut cpu)?, Trap::Host => { let word = cpu.value as u32; - crate::aarch64::host::run(&mut cpu, word); + run_host(&mut cpu, word); cpu.trap = Trap::None; } Trap::Svc => { @@ -155,13 +156,10 @@ fn fetch( if !user_instruction(&instruction) { return Err(crate::aarch64::compile::Error::InvalidUserInstruction); } - if matches!(instruction, Instruction::CacheOp) { + if let Instruction::CacheOp(sys) = &instruction { // The decoder groups EL0 and privileged cache operations together. // Only IC IVAU and DC CVAC/CVAU/CIVAC are permitted in userspace. - let op1 = (word >> 16) & 7; - let crm = (word >> 8) & 15; - let op2 = (word >> 5) & 7; - if op1 != 3 || op2 != 1 || !matches!(crm, 5 | 10 | 11 | 14) { + if sys.op1 != 3 || sys.op2 != 1 || !matches!(sys.crm, 5 | 10 | 11 | 14) { return Err(crate::aarch64::compile::Error::InvalidUserInstruction); } } @@ -187,7 +185,7 @@ fn user_instruction(instruction: &Instruction) -> bool { }, Instruction::Eret | Instruction::Psci - | Instruction::Tlbi + | Instruction::Tlbi(_) | Instruction::Wfi | Instruction::AddressTranslate(_) => false, _ => true, diff --git a/src/ir/Cargo.toml b/src/ir/Cargo.toml index fe6509b..94168d7 100644 --- a/src/ir/Cargo.toml +++ b/src/ir/Cargo.toml @@ -9,6 +9,3 @@ path = "src/lib.rs" [lints] workspace = true - -[dependencies] -pliron = "0.19.0" diff --git a/src/ir/src/attribute.rs b/src/ir/src/attribute.rs index 34d8412..d0143cb 100644 --- a/src/ir/src/attribute.rs +++ b/src/ir/src/attribute.rs @@ -29,6 +29,5 @@ pub enum Attribute { Noreturn, Cold, Align(u32), - Endian(Endianness), Custom(Custom), } diff --git a/src/ir/src/bitcode.rs b/src/ir/src/bitcode.rs index 7a656ca..a296aa7 100644 --- a/src/ir/src/bitcode.rs +++ b/src/ir/src/bitcode.rs @@ -17,10 +17,11 @@ use alloc::vec::Vec; use super::attribute::{AttrValue, Attribute, Custom, Endianness}; use super::function::{ Alloca, Arith, ArithImm, AtomicOp, AtomicOrdering, AtomicRmw, AtomicScope, AttrTarget, Barrier, - BarrierScope, BinOp, Block, Call, CallIndirect, CmpOp, Compare, Convert, Dot, Extract, - Function, GlobalAddr, InputSigns, Inst, Jump, Load, LowFloatConvert, MatMul, MatMulQuant, - MatMulQuantOut, MatMulScale, MatMulType, NvFp4Convert, Opcode, Prefetch, Reduce, Ret, RetPiece, - Select, Splat, Store, StructNew, Terminator, Unary, UnaryOp, VaArg, VaEnd, VaStart, Value, + BarrierScope, BinOp, Block, Call, CallIndirect, CmpOp, Compare, Convert, ConvertKind, Dot, + Extract, Function, GlobalAddr, InputSigns, Inst, Intrinsic, Jump, Load, LowFloatConvert, + MatMul, MatMulQuant, MatMulQuantOut, MatMulScale, MatMulType, MemFlags, NvFp4Convert, Opcode, + Prefetch, Reduce, Ret, RetPiece, Select, Splat, Store, StructNew, Terminator, Unary, UnaryOp, + VaArg, VaEnd, VaStart, Value, }; use super::low_float::Format as LowFloatFormat; use super::nvfp4::ScaleApplication; @@ -42,7 +43,68 @@ impl core::fmt::Display for MalformedSegment { /// Result alias with the error type as a defaulted parameter. pub type Result = core::result::Result; -const MAGIC: &[u8; 4] = b"VBC1"; +/// `VBC3`: integer types carry no signedness (the type record is the tag and the bit width) +/// and signedness lives on the operation, so `BinOp`, `CmpOp`, `AtomicOp`, `Convert` (a kind +/// byte) and `Dot` (a flag byte) have the tag values pinned below. +const MAGIC: &[u8; 4] = b"VBC3"; + +// The wire tag of each operator enum is its discriminant. Pin them at compile time: the +// decoder is a hardcoded switch, so a future reorder must fail to build instead of silently +// turning one operator into another. +const _: () = { + assert!(BinOp::Add as u8 == 0); + assert!(BinOp::Sub as u8 == 1); + assert!(BinOp::Mul as u8 == 2); + assert!(BinOp::Div as u8 == 3); + assert!(BinOp::Rem as u8 == 4); + assert!(BinOp::BitAnd as u8 == 5); + assert!(BinOp::BitOr as u8 == 6); + assert!(BinOp::BitXor as u8 == 7); + assert!(BinOp::Shl as u8 == 8); + assert!(BinOp::Shr as u8 == 9); + assert!(BinOp::Sar as u8 == 10); + assert!(BinOp::UDiv as u8 == 11); + assert!(BinOp::SDiv as u8 == 12); + assert!(BinOp::URem as u8 == 13); + assert!(BinOp::SRem as u8 == 14); + assert!(BinOp::UMulh as u8 == 15); + assert!(BinOp::SMulh as u8 == 16); + assert!(CmpOp::Eq as u8 == 0); + assert!(CmpOp::Ne as u8 == 1); + assert!(CmpOp::Lt as u8 == 2); + assert!(CmpOp::Le as u8 == 3); + assert!(CmpOp::Gt as u8 == 4); + assert!(CmpOp::Ge as u8 == 5); + assert!(CmpOp::Slt as u8 == 6); + assert!(CmpOp::Sle as u8 == 7); + assert!(CmpOp::Sgt as u8 == 8); + assert!(CmpOp::Sge as u8 == 9); + assert!(CmpOp::Ult as u8 == 10); + assert!(CmpOp::Ule as u8 == 11); + assert!(CmpOp::Ugt as u8 == 12); + assert!(CmpOp::Uge as u8 == 13); + assert!(ConvertKind::Trunc as u8 == 0); + assert!(ConvertKind::Zext as u8 == 1); + assert!(ConvertKind::Sext as u8 == 2); + assert!(ConvertKind::SiToFp as u8 == 3); + assert!(ConvertKind::UiToFp as u8 == 4); + assert!(ConvertKind::FpToSi as u8 == 5); + assert!(ConvertKind::FpToUi as u8 == 6); + assert!(ConvertKind::FpResize as u8 == 7); + assert!(AtomicOp::Add as u8 == 0); + assert!(AtomicOp::SMin as u8 == 1); + assert!(AtomicOp::SMax as u8 == 2); + assert!(AtomicOp::BitAnd as u8 == 3); + assert!(AtomicOp::BitOr as u8 == 4); + assert!(AtomicOp::BitXor as u8 == 5); + assert!(AtomicOp::Exchange as u8 == 6); + assert!(AtomicOp::CompareExchange as u8 == 7); + assert!(AtomicOp::UMin as u8 == 8); + assert!(AtomicOp::UMax as u8 == 9); +}; + +/// Flag bits of a `dot` record's flag byte. +const DOT_FLAG_SIGNED: u8 = 1 << 0; /// Bytes the stream header occupies before the type table: the magic, the /// whole-function flag byte, and the fixed-parameter count. @@ -70,7 +132,7 @@ const ATTR_INLINE: u8 = 0; const ATTR_NORETURN: u8 = 1; const ATTR_COLD: u8 = 2; const ATTR_ALIGN: u8 = 3; -const ATTR_ENDIAN: u8 = 4; +// Attribute tag 4 is retired (it was `endian`, now `MemFlags::endian`); 5 stays pinned. const ATTR_CUSTOM: u8 = 5; /// Namespaced attribute payload tags (stable on the wire). @@ -117,6 +179,7 @@ const OP_DEQUANTIZE_NVFP4: u8 = 28; const OP_QUANTIZE_NVFP4: u8 = 29; const OP_REDUCE: u8 = 30; const OP_SPLAT: u8 = 31; +const OP_INTRINSIC: u8 = 32; /// An `atomic_rmw` record flag bit: the compare operand follows the two /// ordinary operand slots. Written from the field and read back into it, so a @@ -124,6 +187,74 @@ const OP_SPLAT: u8 = 31; /// op/compare agreement `verify` enforces. const ATOMIC_FLAG_COMPARE: u8 = 1 << 0; +/// An `intrinsic` record flag bit: the intrinsic has side effects. +const INTRINSIC_FLAG_SIDE_EFFECTS: u8 = 1 << 0; + +// `MemFlags` wire layout, two bytes (tag values pinned; a decoder rejects any other): +// +// byte 0, flags: +// bit 0 volatile +// bits 1..=2 endian: 0 = Native, 1 = Little, 2 = Big (3 invalid) +// bits 3..=5 ordering: 0 = none, otherwise 1 + AtomicOrdering tag +// (1 Relaxed, 2 Acquire, 3 Release, 4 AcqRel, 5 SeqCst; 6, 7 invalid) +// bits 6..=7 reserved, must be zero +// byte 1, align: 0 = no promise, otherwise log2(align) + 1 (so 1..=32) +const MEM_FLAG_VOLATILE: u8 = 1 << 0; +const MEM_ENDIAN_SHIFT: u8 = 1; +const MEM_ORDER_SHIFT: u8 = 3; + +/// Write a `MemFlags`. A non-zero `align` that is not a power of two has no encoding (the +/// verifier rejects it too), so such a function cannot be serialized. +fn write_mem_flags(w: &mut Writer, mem: &MemFlags) -> Result<()> { + let endian: u8 = match mem.endian { + Endianness::Native => 0, + Endianness::Little => 1, + Endianness::Big => 2, + }; + debug_assert_eq!(AtomicOrdering::Relaxed as u8, 0); + debug_assert_eq!(AtomicOrdering::Acquire as u8, 1); + debug_assert_eq!(AtomicOrdering::Release as u8, 2); + debug_assert_eq!(AtomicOrdering::AcqRel as u8, 3); + debug_assert_eq!(AtomicOrdering::SeqCst as u8, 4); + let ordering: u8 = mem.ordering.map_or(0, |o| 1 + o as u8); + let align: u8 = match mem.align { + 0 => 0, + a if a.is_power_of_two() => a.trailing_zeros() as u8 + 1, + _ => return Err(MalformedSegment), + }; + w.u8((mem.volatile as u8 * MEM_FLAG_VOLATILE) + | endian << MEM_ENDIAN_SHIFT + | ordering << MEM_ORDER_SHIFT); + w.u8(align); + Ok(()) +} + +/// Read a `MemFlags`; the bytes come off an UNTRUSTED stream, so every unknown selector is a +/// recoverable fault and never an invalid enum. +fn read_mem_flags(r: &mut Reader<'_>) -> Result { + let flags = r.u8()?; + let align = r.u8()?; + if flags >> 6 != 0 || align > 32 { + return Err(MalformedSegment); + } + let endian = match (flags >> MEM_ENDIAN_SHIFT) & 0b11 { + 0 => Endianness::Native, + 1 => Endianness::Little, + 2 => Endianness::Big, + _ => return Err(MalformedSegment), + }; + let ordering = match (flags >> MEM_ORDER_SHIFT) & 0b111 { + 0 => None, + n => Some(try_atomic_ordering(n - 1)?), + }; + Ok(MemFlags { + volatile: flags & MEM_FLAG_VOLATILE != 0, + align: if align == 0 { 0 } else { 1u32 << (align - 1) }, + ordering, + endian, + }) +} + struct Writer { bytes: Vec, } @@ -229,7 +360,7 @@ pub fn encode(func: &Function) -> Result> { let insts = func.block_insts(block); w.u32(insts.len() as u32); for inst in insts { - write_inst(&mut w, func, *inst, &serial); + write_inst(&mut w, func, *inst, &serial)?; } write_term(&mut w, func, block, &serial); @@ -308,16 +439,6 @@ fn write_attr(w: &mut Writer, attr: &Attribute) { w.u8(ATTR_ALIGN); w.u32(*a); } - Attribute::Endian(e) => { - w.u8(ATTR_ENDIAN); - // The decoder maps this byte back with a range check, so pin the - // tag values here. This follows the `float` and `ptr` arms of - // `write_type`. - debug_assert_eq!(Endianness::Little as u8, 0); - debug_assert_eq!(Endianness::Big as u8, 1); - debug_assert_eq!(Endianness::Native as u8, 2); - w.u8(*e as u8); - } Attribute::Custom(c) => { w.u8(ATTR_CUSTOM); w.string(&c.namespace); @@ -343,7 +464,6 @@ fn write_type(w: &mut Writer, kind: &TypeKind) { TypeKind::Bool => w.u8(0), TypeKind::Int(i) => { w.u8(1); - w.u8(if i.signed { 0 } else { 1 }); w.u16(i.bits); } TypeKind::Float(f) => { @@ -392,7 +512,7 @@ fn write_type(w: &mut Writer, kind: &TypeKind) { } } -fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { +fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) -> Result<()> { let result = func.inst_result(inst); w.u8(if result.is_some() { 1 } else { 0 }); if let Some(r) = result { @@ -452,6 +572,7 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { } Opcode::Convert(c) => { w.u8(OP_CONVERT); + w.u8(c.kind as u8); w.u32(serial[c.value.0 as usize]); } Opcode::DecodeLowFloat(c) => { @@ -526,19 +647,36 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { &c.ret_pieces, ); } - // `volatile` marks an access the optimizer must not remove, move or - // merge. A stream that drops it turns an MMIO register access into an - // ordinary one. + // The `MemFlags` (volatile, byte order, atomic ordering, alignment) change what the + // access means: a stream that drops `volatile` turns an MMIO register access into an + // ordinary one, and one that drops `ordering` or `endian` silently changes the data + // or the memory model. See `write_mem_flags` for the pinned layout. Opcode::Load(l) => { w.u8(OP_LOAD); w.u32(serial[l.ptr.0 as usize]); - w.u8(l.volatile as u8); + write_mem_flags(w, &l.mem)?; } Opcode::Store(st) => { w.u8(OP_STORE); w.u32(serial[st.value.0 as usize]); w.u32(serial[st.ptr.0 as usize]); - w.u8(st.volatile as u8); + write_mem_flags(w, &st.mem)?; + } + // Operands are the arguments, then the immediate and the effect flag. + Opcode::Intrinsic(i) => { + w.u8(OP_INTRINSIC); + w.u32(i.symbol); + let args = func.value_list(i.args); + w.u32(args.len() as u32); + for a in args { + w.u32(serial[a.0 as usize]); + } + w.u64(i.imm as u64); + w.u8(if i.side_effects { + INTRINSIC_FLAG_SIDE_EFFECTS + } else { + 0 + }); } Opcode::Prefetch(pf) => { w.u8(OP_PREFETCH); @@ -572,14 +710,6 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { w.u8(OP_ATOMIC_RMW); // The decoder maps these three bytes back with range checks, so pin // the tag values here. - debug_assert_eq!(AtomicOp::Add as u8, 0); - debug_assert_eq!(AtomicOp::Min as u8, 1); - debug_assert_eq!(AtomicOp::Max as u8, 2); - debug_assert_eq!(AtomicOp::BitAnd as u8, 3); - debug_assert_eq!(AtomicOp::BitOr as u8, 4); - debug_assert_eq!(AtomicOp::BitXor as u8, 5); - debug_assert_eq!(AtomicOp::Exchange as u8, 6); - debug_assert_eq!(AtomicOp::CompareExchange as u8, 7); debug_assert_eq!(AtomicOrdering::Relaxed as u8, 0); debug_assert_eq!(AtomicOrdering::Acquire as u8, 1); debug_assert_eq!(AtomicOrdering::Release as u8, 2); @@ -604,6 +734,7 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { } Opcode::Dot(d) => { w.u8(OP_DOT); + w.u8(if d.signed { DOT_FLAG_SIGNED } else { 0 }); w.u32(serial[d.acc.0 as usize]); w.u32(serial[d.a.0 as usize]); w.u32(serial[d.b.0 as usize]); @@ -649,6 +780,7 @@ fn write_inst(w: &mut Writer, func: &Function, inst: Inst, serial: &[u32]) { w.u8(ga.via_got as u8); } } + Ok(()) } /// Write the quant epilogue, mirroring the Zig field order: relu, out, @@ -967,17 +1099,6 @@ fn read_attr(r: &mut Reader<'_>) -> Result { ATTR_NORETURN => Attribute::Noreturn, ATTR_COLD => Attribute::Cold, ATTR_ALIGN => Attribute::Align(r.u32()?), - ATTR_ENDIAN => { - // The byte comes off an UNTRUSTED stream, so an unknown value is a - // recoverable fault and never an invalid enum. - let order = match r.u8()? { - 0 => Endianness::Little, - 1 => Endianness::Big, - 2 => Endianness::Native, - _ => return Err(MalformedSegment), - }; - Attribute::Endian(order) - } ATTR_CUSTOM => { let namespace = read_string(r)?; let key = read_string(r)?; @@ -1013,9 +1134,8 @@ fn read_type( Ok(match r.u8()? { 0 => func.types.intern(TypeKind::Bool), 1 => { - let signed = r.u8()? == 0; // 0 = signed, 1 = unsigned let bits = r.u16()?; - func.types.intern(TypeKind::Int(IntDesc { signed, bits })) + func.types.intern(TypeKind::Int(IntDesc { bits })) } 2 => { let kind = match r.u8()? { @@ -1241,6 +1361,12 @@ impl Fixup { } Opcode::Reduce(red) => red.vector = next(), Opcode::Splat(sp) => sp.scalar = next(), + Opcode::Intrinsic(i) => { + let args = i.args; + for a in func.value_list_mut(args) { + *a = next(); + } + } Opcode::Matmul(mm) => { mm.a = next(); mm.b = next(); @@ -1391,6 +1517,7 @@ fn requires_result(tag: u8) -> bool { !matches!( tag, OP_STORE + | OP_INTRINSIC | OP_PREFETCH | OP_VA_START | OP_VA_END @@ -1404,7 +1531,7 @@ fn requires_result(tag: u8) -> bool { } /// The result canonicality a record's own has-result type must satisfy: a -/// low-float decode yields f32, an encode yields the unsigned payload width; +/// low-float decode yields f32, an encode yields the payload-width integer; /// anything else is a stream this build cannot read. fn canonical_low_float_result( func: &Function, @@ -1414,16 +1541,16 @@ fn canonical_low_float_result( ) -> bool { match func.types.type_kind(result_type) { TypeKind::Float(f) => decode_direction && *f == FloatKind::F32, - TypeKind::Int(i) => !decode_direction && !i.signed && i.bits == format.payload_bits(), + TypeKind::Int(i) => !decode_direction && i.bits == format.payload_bits(), _ => false, } } -/// Same rule for the NVFP4 records: dequantize yields f32, quantize yields u8. +/// Same rule for the NVFP4 records: dequantize yields f32, quantize yields i8. fn canonical_nvfp4_result(func: &Function, result_type: Type, dequantize_direction: bool) -> bool { match func.types.type_kind(result_type) { TypeKind::Float(f) => dequantize_direction && *f == FloatKind::F32, - TypeKind::Int(i) => !dequantize_direction && !i.signed && i.bits == 8, + TypeKind::Int(i) => !dequantize_direction && i.bits == 8, _ => false, } } @@ -1581,13 +1708,14 @@ fn read_inst( ) } OP_CONVERT => { + let kind = try_convert_kind(r.u8()?)?; fixups.push_slot(r.u32()?); append_res( func, block, serial, rty, - Opcode::Convert(Convert { value: dummy }), + Opcode::Convert(Convert { value: dummy, kind }), ) } OP_DECODE_LOW_FLOAT | OP_ENCODE_LOW_FLOAT => { @@ -1717,32 +1845,61 @@ fn read_inst( } OP_LOAD => { fixups.push_slot(r.u32()?); - let is_volatile = r.u8()? != 0; + let mem = read_mem_flags(r)?; append_res( func, block, serial, rty, - Opcode::Load(Load { - ptr: dummy, - volatile: is_volatile, - }), + Opcode::Load(Load { ptr: dummy, mem }), ) } OP_STORE => { for _ in 0..2 { fixups.push_slot(r.u32()?); } - let is_volatile = r.u8()? != 0; + let mem = read_mem_flags(r)?; func.append_stmt_raw( block, Opcode::Store(Store { value: dummy, ptr: dummy, - volatile: is_volatile, + mem, }), ) } + OP_INTRINSIC => { + let symbol = r.u32()?; + // The stream is UNTRUSTED: the name must exist in the symbol table. + if symbol as usize >= func.symbol_count() { + return Err(MalformedSegment); + } + let n = r.u32()? as usize; + if n > r.remaining() { + return Err(MalformedSegment); + } + for _ in 0..n { + fixups.push_slot(r.u32()?); + } + let imm = r.u64()? as i64; + let flags = r.u8()?; + if flags & !INTRINSIC_FLAG_SIDE_EFFECTS != 0 { + return Err(MalformedSegment); + } + let dummies = alloc::vec![dummy; n]; + let op = Opcode::Intrinsic(Intrinsic { + symbol, + args: func.intern_values(&dummies), + imm, + side_effects: flags & INTRINSIC_FLAG_SIDE_EFFECTS != 0, + }); + // The result is OPTIONAL, exactly as for a `call`. + if has_result { + append_res(func, block, serial, rty, op) + } else { + func.append_stmt_raw(block, op) + } + } OP_PREFETCH => { fixups.push_slot(r.u32()?); func.append_stmt_raw(block, Opcode::Prefetch(Prefetch { ptr: dummy })) @@ -1822,6 +1979,10 @@ fn read_inst( } } OP_DOT => { + let flags = r.u8()?; + if flags & !DOT_FLAG_SIGNED != 0 { + return Err(MalformedSegment); + } for _ in 0..3 { fixups.push_slot(r.u32()?); } @@ -1834,6 +1995,7 @@ fn read_inst( acc: dummy, a: dummy, b: dummy, + signed: flags & DOT_FLAG_SIGNED != 0, }), ) } @@ -1982,13 +2144,15 @@ fn read_inst( fn try_atomic_op(raw: u8) -> Result { Ok(match raw { 0 => AtomicOp::Add, - 1 => AtomicOp::Min, - 2 => AtomicOp::Max, + 1 => AtomicOp::SMin, + 2 => AtomicOp::SMax, 3 => AtomicOp::BitAnd, 4 => AtomicOp::BitOr, 5 => AtomicOp::BitXor, 6 => AtomicOp::Exchange, 7 => AtomicOp::CompareExchange, + 8 => AtomicOp::UMin, + 9 => AtomicOp::UMax, _ => return Err(MalformedSegment), }) } @@ -2026,7 +2190,13 @@ fn try_bin_op(raw: u8) -> Result { 7 => BinOp::BitXor, 8 => BinOp::Shl, 9 => BinOp::Shr, - 10 => BinOp::Mulh, + 10 => BinOp::Sar, + 11 => BinOp::UDiv, + 12 => BinOp::SDiv, + 13 => BinOp::URem, + 14 => BinOp::SRem, + 15 => BinOp::UMulh, + 16 => BinOp::SMulh, _ => return Err(MalformedSegment), }) } @@ -2039,6 +2209,28 @@ fn try_cmp_op(raw: u8) -> Result { 3 => CmpOp::Le, 4 => CmpOp::Gt, 5 => CmpOp::Ge, + 6 => CmpOp::Slt, + 7 => CmpOp::Sle, + 8 => CmpOp::Sgt, + 9 => CmpOp::Sge, + 10 => CmpOp::Ult, + 11 => CmpOp::Ule, + 12 => CmpOp::Ugt, + 13 => CmpOp::Uge, + _ => return Err(MalformedSegment), + }) +} + +fn try_convert_kind(raw: u8) -> Result { + Ok(match raw { + 0 => ConvertKind::Trunc, + 1 => ConvertKind::Zext, + 2 => ConvertKind::Sext, + 3 => ConvertKind::SiToFp, + 4 => ConvertKind::UiToFp, + 5 => ConvertKind::FpToSi, + 6 => ConvertKind::FpToUi, + 7 => ConvertKind::FpResize, _ => return Err(MalformedSegment), }) } @@ -2107,7 +2299,7 @@ mod tests { /// The stream header a hand-built test module starts with: the magic, no /// whole-function flags, and a zero fixed-parameter count. - const TEST_HEADER: &[u8] = b"VBC1\x00\x00\x00\x00\x00"; + const TEST_HEADER: &[u8] = b"VBC3\x00\x00\x00\x00\x00"; /// A structural printer for the oracle: renders a function's blocks, /// params, instructions (operands numbered by canonical serial) and @@ -2173,13 +2365,40 @@ mod tests { vn(c.lhs), vn(c.rhs) )), - Opcode::Convert(c) => out.push_str(&format!("convert {}", vn(c.value))), + Opcode::Convert(c) => { + out.push_str(&format!("convert {} {}", c.kind.name(), vn(c.value))) + } + Opcode::Dot(d) => out.push_str(&format!( + "dot {} {} {} {}", + if d.signed { "signed" } else { "unsigned" }, + vn(d.acc), + vn(d.a), + vn(d.b) + )), Opcode::Call(c) => { out.push_str(&format!("call \"{}\"", func.symbol_name(c.symbol))); for a in func.value_list(c.args) { out.push_str(&format!(" {}", vn(*a))); } } + Opcode::Load(l) => out.push_str(&format!("load {} {:?}", vn(l.ptr), l.mem)), + Opcode::Store(st) => out.push_str(&format!( + "store {} {} {:?}", + vn(st.value), + vn(st.ptr), + st.mem + )), + Opcode::Intrinsic(i) => { + out.push_str(&format!( + "intrinsic \"{}\" imm {} effects {}", + func.symbol_name(i.symbol), + i.imm, + i.side_effects + )); + for a in func.value_list(i.args) { + out.push_str(&format!(" {}", vn(*a))); + } + } Opcode::If(cf) => out.push_str(&format!( "if {} -> block{}, else -> block{}", vn(cf.cond), @@ -2242,10 +2461,7 @@ mod tests { // f(x, y): if x a, _ => unreachable!(), }; - assert_eq!(AtomicOp::Min, reading.op); + assert_eq!(AtomicOp::SMin, reading.op); assert_eq!(AtomicOrdering::AcqRel, reading.ordering); assert_eq!(AtomicScope::Device, reading.scope); assert_eq!(None, reading.compare); @@ -2480,12 +2693,290 @@ mod tests { } #[test] - fn round_trips_a_compare_exchange_and_its_compare_operand() { + fn the_operator_tags_are_pinned_on_the_wire() { + let bins = [ + (0, BinOp::Add), + (1, BinOp::Sub), + (2, BinOp::Mul), + (3, BinOp::Div), + (4, BinOp::Rem), + (5, BinOp::BitAnd), + (6, BinOp::BitOr), + (7, BinOp::BitXor), + (8, BinOp::Shl), + (9, BinOp::Shr), + (10, BinOp::Sar), + (11, BinOp::UDiv), + (12, BinOp::SDiv), + (13, BinOp::URem), + (14, BinOp::SRem), + (15, BinOp::UMulh), + (16, BinOp::SMulh), + ]; + for (tag, op) in bins { + assert_eq!(op as u8, tag, "{op:?}"); + assert_eq!(try_bin_op(tag), Ok(op)); + } + assert!(try_bin_op(17).is_err()); + + let cmps = [ + (0, CmpOp::Eq), + (1, CmpOp::Ne), + (2, CmpOp::Lt), + (3, CmpOp::Le), + (4, CmpOp::Gt), + (5, CmpOp::Ge), + (6, CmpOp::Slt), + (7, CmpOp::Sle), + (8, CmpOp::Sgt), + (9, CmpOp::Sge), + (10, CmpOp::Ult), + (11, CmpOp::Ule), + (12, CmpOp::Ugt), + (13, CmpOp::Uge), + ]; + for (tag, op) in cmps { + assert_eq!(op as u8, tag, "{op:?}"); + assert_eq!(try_cmp_op(tag), Ok(op)); + } + assert!(try_cmp_op(14).is_err()); + + let kinds = [ + (0, ConvertKind::Trunc), + (1, ConvertKind::Zext), + (2, ConvertKind::Sext), + (3, ConvertKind::SiToFp), + (4, ConvertKind::UiToFp), + (5, ConvertKind::FpToSi), + (6, ConvertKind::FpToUi), + (7, ConvertKind::FpResize), + ]; + for (tag, kind) in kinds { + assert_eq!(kind as u8, tag, "{kind:?}"); + assert_eq!(try_convert_kind(tag), Ok(kind)); + } + assert!(try_convert_kind(8).is_err()); + + let atomics = [ + (0, AtomicOp::Add), + (1, AtomicOp::SMin), + (2, AtomicOp::SMax), + (3, AtomicOp::BitAnd), + (4, AtomicOp::BitOr), + (5, AtomicOp::BitXor), + (6, AtomicOp::Exchange), + (7, AtomicOp::CompareExchange), + (8, AtomicOp::UMin), + (9, AtomicOp::UMax), + ]; + for (tag, op) in atomics { + assert_eq!(op as u8, tag, "{op:?}"); + assert_eq!(try_atomic_op(tag), Ok(op)); + } + assert!(try_atomic_op(10).is_err()); + } + + /// Appends one instruction of every signedness-carrying form to a single block. + fn every_operator_function() -> Function { let mut func = Function::new(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, + let i8_t = func.types.intern(TypeKind::Int(IntDesc { bits: 8 })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let i64_t = func.types.intern(TypeKind::Int(IntDesc { bits: 64 })); + let f32_t = func.types.intern(TypeKind::Float(FloatKind::F32)); + let f64_t = func.types.intern(TypeKind::Float(FloatKind::F64)); + let bool_t = func.types.intern(TypeKind::Bool); + let ptr_t = func.types.ptr_global(); + let v16i8 = func.types.intern(TypeKind::Vector(VectorDesc { + len: 16, + elem: i8_t, + })); + let v4i32 = func.types.intern(TypeKind::Vector(VectorDesc { + len: 4, + elem: i32_t, })); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let b = func.append_block_param(entry, i32_t); + let x = func.append_block_param(entry, f32_t); + let y = func.append_block_param(entry, f32_t); + let p = func.append_block_param(entry, ptr_t); + let acc = func.append_block_param(entry, v4i32); + let va = func.append_block_param(entry, v16i8); + let vb = func.append_block_param(entry, v16i8); + let flag = func.append_block_param(entry, bool_t); + let narrow = func.append_block_param(entry, i8_t); + let double = func.append_block_param(entry, f64_t); + + for op in [ + BinOp::Add, + BinOp::Sub, + BinOp::Mul, + BinOp::BitAnd, + BinOp::BitOr, + BinOp::BitXor, + BinOp::Shl, + BinOp::Shr, + BinOp::Sar, + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::UMulh, + BinOp::SMulh, + ] { + func.append_inst(entry, i32_t, Opcode::Arith(Arith { op, lhs: a, rhs: b })); + func.append_arith_imm(entry, i32_t, op, a, 5); + } + for op in [BinOp::Div, BinOp::Rem] { + func.append_inst(entry, f32_t, Opcode::Arith(Arith { op, lhs: x, rhs: y })); + } + for op in [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Slt, + CmpOp::Sle, + CmpOp::Sgt, + CmpOp::Sge, + ] + .into_iter() + .chain([CmpOp::Ult, CmpOp::Ule, CmpOp::Ugt, CmpOp::Uge]) + { + func.append_inst(entry, bool_t, Opcode::Icmp(Compare { op, lhs: a, rhs: b })); + } + for op in [CmpOp::Lt, CmpOp::Le, CmpOp::Gt, CmpOp::Ge] { + func.append_inst(entry, bool_t, Opcode::Icmp(Compare { op, lhs: x, rhs: y })); + } + func.append_convert(entry, i8_t, ConvertKind::Trunc, a); + func.append_convert(entry, i32_t, ConvertKind::Zext, flag); + func.append_convert(entry, i64_t, ConvertKind::Zext, a); + func.append_convert(entry, i64_t, ConvertKind::Sext, a); + func.append_convert(entry, f32_t, ConvertKind::SiToFp, a); + func.append_convert(entry, f32_t, ConvertKind::UiToFp, narrow); + func.append_convert(entry, i32_t, ConvertKind::FpToSi, x); + func.append_convert(entry, i64_t, ConvertKind::FpToUi, double); + func.append_convert(entry, f64_t, ConvertKind::FpResize, x); + func.append_dot(entry, acc, va, vb, true); + func.append_dot(entry, acc, va, vb, false); + for op in [ + AtomicOp::Add, + AtomicOp::SMin, + AtomicOp::SMax, + AtomicOp::UMin, + AtomicOp::UMax, + AtomicOp::BitAnd, + AtomicOp::BitOr, + AtomicOp::BitXor, + AtomicOp::Exchange, + AtomicOp::CompareExchange, + ] { + func.append_atomic_rmw( + entry, + AtomicRmw { + op, + ptr: p, + value: a, + compare: (op == AtomicOp::CompareExchange).then_some(b), + ordering: AtomicOrdering::SeqCst, + scope: AtomicScope::Device, + }, + ); + } + func.set_terminator(entry, Terminator::Ret(Ret::none())); + func + } + + #[test] + fn every_operator_round_trips_field_by_field() { + let func = every_operator_function(); + let decoded = expect_round_trip(&func); + let entry = Block(0); + let (want, got) = (func.block_insts(entry), decoded.block_insts(entry)); + assert_eq!(want.len(), got.len()); + for (w, g) in want.iter().zip(got) { + // Values are numbered by the canonical walk, which here is creation order. + assert_eq!(func.opcode(*w), decoded.opcode(*g)); + let (wt, gt) = ( + func.inst_result(*w).map(|r| func.value_type(r)), + decoded.inst_result(*g).map(|r| decoded.value_type(r)), + ); + let text = |f: &Function, t: Option| t.map(|t| format!("{}", f.types.display(t))); + assert_eq!(text(&func, wt), text(&decoded, gt)); + } + } + + #[test] + fn integer_type_records_have_no_signedness_byte() { + // tag, then the width: `i32` is exactly `[1, 32, 0]`. + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let entry = func.append_block(); + func.append_block_param(entry, i32_t); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + let bytes = encode(&func).unwrap(); + assert_eq!(&bytes[..4], b"VBC3"); + assert_eq!( + &bytes[HEADER_LEN..HEADER_LEN + 4], + &[1, 0, 0, 0], + "one type" + ); + assert_eq!(&bytes[HEADER_LEN + 4..HEADER_LEN + 7], &[1, 32, 0]); + assert!(decode(&bytes).is_ok()); + } + + #[test] + fn a_stream_with_the_previous_magic_is_rejected() { + let func = every_operator_function(); + let mut bytes = encode(&func).unwrap(); + assert!(decode(&bytes).is_ok()); + bytes[..4].copy_from_slice(b"VBC2"); + assert!(decode(&bytes).is_err()); + } + + #[test] + fn an_unknown_convert_kind_or_dot_flag_is_malformed() { + let mut func = Function::new(); + let i8_t = func.types.intern(TypeKind::Int(IntDesc { bits: 8 })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let v4i32 = func.types.intern(TypeKind::Vector(VectorDesc { + len: 4, + elem: i32_t, + })); + let entry = func.append_block(); + let a = func.append_block_param(entry, i8_t); + let acc = func.append_block_param(entry, v4i32); + func.append_convert(entry, i32_t, ConvertKind::Zext, a); + func.append_dot(entry, acc, acc, acc, true); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + let bytes = encode(&func).unwrap(); + assert!(decode(&bytes).is_ok()); + + // The first operand of the convert is value 0, of the dot value 1. + let convert_at = bytes + .windows(6) + .position(|w| w == [OP_CONVERT, ConvertKind::Zext as u8, 0, 0, 0, 0]) + .expect("convert record"); + let mut bad = bytes.clone(); + bad[convert_at + 1] = 8; + assert!(decode(&bad).is_err()); + + let dot_at = bytes + .windows(6) + .position(|w| w == [OP_DOT, DOT_FLAG_SIGNED, 1, 0, 0, 0]) + .expect("dot record"); + let mut bad = bytes.clone(); + bad[dot_at + 1] = DOT_FLAG_SIGNED | 0x80; + assert!(decode(&bad).is_err()); + let mut unsigned = bytes.clone(); + unsigned[dot_at + 1] = 0; + let back = decode(&unsigned).unwrap(); + let last = *back.block_insts(entry).last().unwrap(); + assert!(matches!(back.opcode(last), Opcode::Dot(d) if !d.signed)); + } + + #[test] + fn round_trips_a_compare_exchange_and_its_compare_operand() { + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let ptr_t = func.types.ptr_global(); let entry = func.append_block(); let p = func.append_block_param(entry, ptr_t); @@ -2524,10 +3015,7 @@ mod tests { #[test] fn an_unknown_atomic_selector_byte_is_rejected() { let mut func = Function::new(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let ptr_t = func.types.ptr_global(); let entry = func.append_block(); let p = func.append_block_param(entry, ptr_t); @@ -2631,17 +3119,14 @@ mod tests { #[test] fn rejects_truncated_bitcode() { - assert!(decode(b"VBC1\x01").is_err()); + assert!(decode(b"VBC3\x01").is_err()); assert!(decode(b"nope").is_err()); } #[test] fn round_trips_va_start_va_arg_va_end_through_bitcode() { let mut func = Function::new(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let ptr_t = func.types.ptr_global(); let entry = func.append_block(); let list = func.append_block_param(entry, ptr_t); @@ -2719,17 +3204,17 @@ mod tests { // An unknown arith operator byte, no such BinOp. let bad_arith = bconcat![ TEST_HEADER, - [1, 0, 0, 0], // type_count = 1 - [1, 0, 32, 0], // int, signed, 32 bits - [0, 0, 0, 0], // sym_count = 0 - [1, 0, 0, 0], // block_count = 1 - [1, 0, 0, 0], // block 0 param_count = 1 - [0, 0, 0, 0], // param 0 type 0 - [1, 0, 0, 0], // inst_count = 1 - [1], // has_result = 1 - [0, 0, 0, 0], // result type 0 - [2], // tag = op_arith - [0xff], // operator byte: no such BinOp + [1, 0, 0, 0], // type_count = 1 + [1, 32, 0], // int, 32 bits + [0, 0, 0, 0], // sym_count = 0 + [1, 0, 0, 0], // block_count = 1 + [1, 0, 0, 0], // block 0 param_count = 1 + [0, 0, 0, 0], // param 0 type 0 + [1, 0, 0, 0], // inst_count = 1 + [1], // has_result = 1 + [0, 0, 0, 0], // result type 0 + [2], // tag = op_arith + [0xff], // operator byte: no such BinOp ]; assert!(decode(&bad_arith[..]).is_err()); @@ -2812,10 +3297,7 @@ mod tests { let mut func = Function::new(); let s = func.types.intern(TypeKind::Ptr(AddressSpace::Shared)); let g = func.types.ptr_global(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let b = func.append_block(); let ps = func.append_block_param(b, s); let pg = func.append_block_param(b, g); @@ -2838,10 +3320,7 @@ mod tests { #[test] fn a_two_value_ret_round_trips_through_bitcode() { let mut func = Function::new(); - let i32_t = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let entry = func.append_block(); let a = func.append_block_param(entry, i32_t); let b = func.append_block_param(entry, i32_t); @@ -2859,4 +3338,163 @@ mod tests { assert_eq!(params[1], decoded_ret.values[1]); let _ = expect_round_trip(&func); } + + fn all_mem_flags() -> Vec { + let orderings = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Acquire), + Some(AtomicOrdering::Release), + Some(AtomicOrdering::AcqRel), + Some(AtomicOrdering::SeqCst), + ]; + let endians = [Endianness::Native, Endianness::Little, Endianness::Big]; + let mut out = Vec::new(); + for volatile in [false, true] { + for align in [0u32, 1, 2, 8, 4096, 1 << 31] { + for ordering in orderings { + for endian in endians { + out.push(MemFlags { + volatile, + align, + ordering, + endian, + }); + } + } + } + } + out + } + + #[test] + fn loads_and_stores_with_every_mem_flag_round_trip_through_bitcode() { + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let mut expected = Vec::new(); + let mut last = None; + for mem in all_mem_flags() { + let v = func.append_load(entry, i32_t, p, mem); + func.append_store_mem(entry, v, p, mem); + expected.push(mem); + expected.push(mem); + last = Some(v); + } + func.set_terminator(entry, Terminator::Ret(Ret::one(last.unwrap()))); + + let decoded = expect_round_trip(&func); + let got: Vec = decoded + .block_insts(entry) + .iter() + .map(|&i| match decoded.opcode(i) { + Opcode::Load(l) => l.mem, + Opcode::Store(st) => st.mem, + _ => unreachable!(), + }) + .collect(); + assert_eq!(got, expected); + } + + #[test] + fn mem_flags_wire_layout_is_pinned() { + let encoded = |mem: MemFlags| { + let mut w = Writer::new(); + write_mem_flags(&mut w, &mem).unwrap(); + w.bytes + }; + assert_eq!(encoded(MemFlags::new()), [0, 0]); + assert_eq!( + encoded(MemFlags { + volatile: true, + align: 8, + ordering: Some(AtomicOrdering::SeqCst), + endian: Endianness::Big, + }), + [1 | (2 << 1) | (5 << 3), 4] + ); + assert_eq!( + encoded(MemFlags { + endian: Endianness::Little, + ordering: Some(AtomicOrdering::Relaxed), + align: 1, + ..MemFlags::new() + }), + [(1 << 1) | (1 << 3), 1] + ); + } + + #[test] + fn malformed_mem_flags_are_rejected() { + let read = |bytes: &[u8]| read_mem_flags(&mut Reader { bytes, pos: 0 }); + assert!(read(&[0, 0]).is_ok()); + assert!(read(&[3 << 1, 0]).is_err(), "endian selector 3"); + assert!(read(&[6 << 3, 0]).is_err(), "ordering selector 6"); + assert!(read(&[7 << 3, 0]).is_err(), "ordering selector 7"); + assert!(read(&[1 << 6, 0]).is_err(), "reserved flag bit"); + assert!(read(&[0, 33]).is_err(), "align code past 2^31"); + assert!(read(&[0]).is_err(), "truncated"); + let mut w = Writer::new(); + let odd = MemFlags { + align: 3, + ..MemFlags::new() + }; + assert!(write_mem_flags(&mut w, &odd).is_err()); + } + + #[test] + fn a_non_power_of_two_align_fails_to_encode() { + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let v = func.append_load( + entry, + i32_t, + p, + MemFlags { + align: 6, + ..MemFlags::new() + }, + ); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + assert!(encode(&func).is_err()); + } + + #[test] + fn intrinsics_round_trip_through_bitcode() { + let mut func = Function::new(); + let i32_t = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let b = func.append_block_param(entry, i32_t); + let valued = func.append_intrinsic(entry, i32_t, "aarch64.aese", &[a, b], -7, true); + let pure = func.append_intrinsic(entry, i32_t, "x86.bswap", &[valued], 0, false); + func.append_intrinsic_void(entry, "arm.dmb", &[], 15, true); + func.append_intrinsic_void(entry, "nop.pure", &[a], i64::MIN, false); + func.set_terminator(entry, Terminator::Ret(Ret::one(pure))); + + let decoded = expect_round_trip(&func); + let insts = decoded.block_insts(entry).to_vec(); + assert_eq!(insts.len(), 4); + let get = |n: usize| match decoded.opcode(insts[n]) { + Opcode::Intrinsic(i) => i, + _ => panic!("expected intrinsic"), + }; + let params = decoded.block_params(entry).to_vec(); + let v0 = decoded.inst_result(insts[0]).unwrap(); + assert_eq!(decoded.symbol_name(get(0).symbol), "aarch64.aese"); + assert_eq!(decoded.value_list(get(0).args), ¶ms[..]); + assert_eq!((get(0).imm, get(0).side_effects), (-7, true)); + assert_eq!(decoded.value_list(get(1).args), &[v0]); + assert!(!get(1).side_effects && decoded.inst_result(insts[1]).is_some()); + assert_eq!(decoded.symbol_name(get(2).symbol), "arm.dmb"); + assert!(decoded.inst_result(insts[2]).is_none() && get(2).imm == 15); + assert_eq!(get(3).imm, i64::MIN); + assert!(!get(3).side_effects); + assert_eq!(decoded.value_list(get(3).args), ¶ms[..1]); + } } diff --git a/src/ir/src/builder.rs b/src/ir/src/builder.rs index 6a9f0aa..efa9e9f 100644 --- a/src/ir/src/builder.rs +++ b/src/ir/src/builder.rs @@ -1,6 +1,6 @@ extern crate alloc; -use super::function::{BinOp, Block, CmpOp, EdgeDesc, Function, Value}; +use super::function::{BinOp, Block, CmpOp, ConvertKind, EdgeDesc, Function, MemFlags, Value}; use super::types::Type; pub struct Builder<'a> { @@ -44,6 +44,11 @@ impl<'a> Builder<'a> { ) } + /// `convert , value`; see [`ConvertKind`] for the legal type pairs. + pub fn convert(&mut self, ty: Type, kind: ConvertKind, value: Value) -> Value { + self.func.append_convert(self.block, ty, kind, value) + } + pub fn select(&mut self, cond: Value, then_value: Value, else_value: Value) -> Value { let ty = self.func.value_type(then_value); self.func.append_inst( @@ -76,20 +81,38 @@ impl<'a> Builder<'a> { } pub fn load(&mut self, ty: Type, ptr: Value) -> Value { - self.func.append_inst( - self.block, - ty, - super::function::Opcode::Load(super::function::Load { - ptr, - volatile: false, - }), - ) + self.load_mem(ty, ptr, MemFlags::new()) + } + + pub fn load_mem(&mut self, ty: Type, ptr: Value, mem: MemFlags) -> Value { + self.func.append_load(self.block, ty, ptr, mem) } pub fn store(&mut self, value: Value, ptr: Value) { self.func.append_store(self.block, value, ptr); } + pub fn store_mem(&mut self, value: Value, ptr: Value, mem: MemFlags) { + self.func.append_store_mem(self.block, value, ptr, mem); + } + + pub fn intrinsic( + &mut self, + ty: Type, + name: &str, + args: &[Value], + imm: i64, + side_effects: bool, + ) -> Value { + self.func + .append_intrinsic(self.block, ty, name, args, imm, side_effects) + } + + pub fn intrinsic_void(&mut self, name: &str, args: &[Value], imm: i64, side_effects: bool) { + self.func + .append_intrinsic_void(self.block, name, args, imm, side_effects); + } + pub fn ret(&mut self, value: Option) { let r = match value { Some(v) => super::function::Ret::one(v), @@ -122,10 +145,7 @@ mod tests { use super::*; fn i32t(f: &mut Function) -> Type { - f.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })) + f.types.intern(TypeKind::Int(IntDesc { bits: 32 })) } #[test] @@ -173,10 +193,29 @@ mod tests { let a = func.append_block_param(entry, i32_t); let b = func.append_block_param(entry, i32_t); let mut bld = Builder::new(&mut func, entry); - let c = bld.icmp(CmpOp::Gt, a, b); + let c = bld.icmp(CmpOp::Sgt, a, b); assert_eq!(bool_t, func.value_type(c)); } + #[test] + fn convert_takes_the_result_type_and_kind() { + let mut func = Function::new(); + let i32_t = i32t(&mut func); + let i64_t = func.types.intern(TypeKind::Int(IntDesc { bits: 64 })); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let mut bld = Builder::new(&mut func, entry); + let wide = bld.convert(i64_t, ConvertKind::Sext, a); + assert_eq!(i64_t, func.value_type(wide)); + match func.opcode(func.defining_inst(wide).unwrap()) { + super::super::function::Opcode::Convert(c) => { + assert_eq!(a, c.value); + assert_eq!(ConvertKind::Sext, c.kind); + } + _ => panic!("expected convert"), + } + } + #[test] fn struct_construction_and_extraction() { let mut func = Function::new(); diff --git a/src/ir/src/dialect/attrs.rs b/src/ir/src/dialect/attrs.rs deleted file mode 100644 index 2e4eada..0000000 --- a/src/ir/src/dialect/attrs.rs +++ /dev/null @@ -1,726 +0,0 @@ -//! Operation payload attributes for the `volt` dialect. -//! -//! Small enums reuse the existing volt definitions (from -//! [`function`](crate::function), [`low_float`](crate::low_float), -//! [`nvfp4`](crate::nvfp4), and [`attribute`](crate::attribute)); the -//! structs below wrap them as pliron attributes with keyword syntax. - -use alloc::string::{String, ToString}; -use alloc::vec::Vec; - -use pliron::{ - combine::{ - Parser, choice, - parser::{ - char::{spaces, string}, - token::{satisfy, token}, - }, - }, - context::Context, - derive::pliron_attr, - irfmt::parsers::int_parser, - parsable::{Parsable, ParseResult, StateStream}, - printable::{self, Printable}, -}; - -use super::super::attribute::{AttrValue, Attribute, Custom, Endianness}; -use super::super::function::{ - AtomicOp, AtomicOrdering, AtomicScope, BarrierScope, BinOp, CmpOp, InputSigns, MatMulQuantOut, - MatMulType, RetPiece, UnaryOp, -}; -use super::super::low_float::Format as LowFloatFormat; -use super::super::nvfp4::ScaleApplication; - -/// Parse one of several keywords. -/// Parse one of several keywords. -fn keyword<'a, T: Copy>( - state_stream: &mut StateStream<'a>, - options: &[(&'static str, T)], -) -> ParseResult<'a, T> { - let mut parsers = Vec::new(); - for (word, value) in options { - parsers.push(string(*word).map(move |_| *value)); - } - spaces() - .with(choice(&mut parsers[..])) - .parse_stream(state_stream) - .into() -} - -macro_rules! keyword_attr { - ($attr:ident, $inner:ty, $name:literal, [$($word:literal => $variant:path),* $(,)?]) => { - #[doc = concat!("A `", $name, "` attribute.")] - #[pliron_attr(name = $name, verifier = "succ")] - #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] - pub struct $attr(pub $inner); - - impl $attr { - /// Wrap a value. - pub fn new(value: $inner) -> Self { - Self(value) - } - - /// The wrapped value. - pub fn value(&self) -> $inner { - self.0 - } - } - - impl Printable for $attr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let word = match self.0 { - $($variant => $word,)* - }; - f.write_str(word) - } - } - - impl Parsable for $attr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - keyword(state_stream, &[$(($word, $variant),)*]) - .map(|(val, rest)| (Self(val), rest)) - } - } - }; -} - -keyword_attr!( - BinOpAttr, - BinOp, - "volt.bin_op", - [ - "add" => BinOp::Add, "sub" => BinOp::Sub, "mul" => BinOp::Mul, - "div" => BinOp::Div, "rem" => BinOp::Rem, "and" => BinOp::BitAnd, - "or" => BinOp::BitOr, "xor" => BinOp::BitXor, "shl" => BinOp::Shl, - "shr" => BinOp::Shr, "mulh" => BinOp::Mulh, - ] -); - -keyword_attr!( - CmpOpAttr, - CmpOp, - "volt.cmp_op", - [ - "eq" => CmpOp::Eq, "ne" => CmpOp::Ne, "lt" => CmpOp::Lt, - "le" => CmpOp::Le, "gt" => CmpOp::Gt, "ge" => CmpOp::Ge, - ] -); - -keyword_attr!( - UnaryOpAttr, - UnaryOp, - "volt.unary_op", - [ - "reinterpret" => UnaryOp::Reinterpret, "sqrt" => UnaryOp::Sqrt, - "ceil" => UnaryOp::Ceil, "floor" => UnaryOp::Floor, - "trunc" => UnaryOp::Trunc, "nearest" => UnaryOp::Nearest, - ] -); - -keyword_attr!( - BarrierAttr, - BarrierScope, - "volt.barrier_scope", - [ - "workgroup" => BarrierScope::Workgroup, - "subgroup" => BarrierScope::Subgroup, - ] -); - -keyword_attr!( - AtomicOpAttr, - AtomicOp, - "volt.atomic_op", - [ - "add" => AtomicOp::Add, "min" => AtomicOp::Min, - "max" => AtomicOp::Max, "and" => AtomicOp::BitAnd, - "or" => AtomicOp::BitOr, "xor" => AtomicOp::BitXor, - "exchange" => AtomicOp::Exchange, - "compare_exchange" => AtomicOp::CompareExchange, - ] -); - -keyword_attr!( - AtomicOrderingAttr, - AtomicOrdering, - "volt.atomic_ordering", - [ - "relaxed" => AtomicOrdering::Relaxed, - "acquire" => AtomicOrdering::Acquire, - "release" => AtomicOrdering::Release, - "acq_rel" => AtomicOrdering::AcqRel, - "seq_cst" => AtomicOrdering::SeqCst, - ] -); - -keyword_attr!( - AtomicScopeAttr, - AtomicScope, - "volt.atomic_scope", - [ - "workgroup" => AtomicScope::Workgroup, - "device" => AtomicScope::Device, - "system" => AtomicScope::System, - ] -); - -keyword_attr!( - LowFloatAttr, - LowFloatFormat, - "volt.low_float_format", - [ - "bf16" => LowFloatFormat::Bf16, - "f8e4m3" => LowFloatFormat::F8E4M3, - "f8e5m2" => LowFloatFormat::F8E5M2, - ] -); - -keyword_attr!( - ScaleAppAttr, - ScaleApplication, - "volt.scale_application", - [ - "multiply" => ScaleApplication::Multiply, - "divide" => ScaleApplication::Divide, - ] -); - -keyword_attr!( - MatMulTypeAttr, - MatMulType, - "volt.matmul_dtype", - [ - "fp32" => MatMulType::Fp32, "fp16" => MatMulType::Fp16, - "int8" => MatMulType::Int8, "uint8" => MatMulType::Uint8, - ] -); - -keyword_attr!( - MatMulQuantOutAttr, - MatMulQuantOut, - "volt.matmul_quant_out", - [ - "i8" => MatMulQuantOut::I8, - "u8" => MatMulQuantOut::U8, - ] -); - -/// A generic signed integer attribute: immediates, indices, counts. -#[pliron_attr(name = "volt.imm", verifier = "succ")] -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct ImmAttr(pub i64); - -impl ImmAttr { - /// Wrap a value. - pub fn new(value: i64) -> Self { - Self(value) - } - - /// The wrapped value. - pub fn value(&self) -> i64 { - self.0 - } -} - -impl Printable for ImmAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl Parsable for ImmAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(int_parser::()) - .map(Self) - .parse_stream(state_stream) - .into() - } -} - -/// Raw bits of an `f64` constant, printed as decimal. -#[pliron_attr(name = "volt.fbits", verifier = "succ")] -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct FloatBitsAttr(pub u64); - -impl FloatBitsAttr { - /// Wrap an `f64` constant. - pub fn new(value: f64) -> Self { - Self(value.to_bits()) - } - - /// The constant value. - pub fn value(&self) -> f64 { - f64::from_bits(self.0) - } -} - -impl Printable for FloatBitsAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl Parsable for FloatBitsAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(int_parser::()) - .map(Self) - .parse_stream(state_stream) - .into() - } -} - -/// Raw bits of an `f128` constant, printed as decimal. -#[pliron_attr(name = "volt.f128bits", verifier = "succ")] -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] -pub struct F128BitsAttr(pub u128); - -impl F128BitsAttr { - /// Wrap a value. - pub fn new(value: u128) -> Self { - Self(value) - } - - /// The wrapped value. - pub fn value(&self) -> u128 { - self.0 - } -} - -impl Printable for F128BitsAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl Parsable for F128BitsAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(int_parser::()) - .map(Self) - .parse_stream(state_stream) - .into() - } -} - -/// Owned matrix-multiply configuration (mirrors [`MatMul`](crate::function::MatMul)). -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct MatMulDesc { - /// Rows of A / C. - pub m: u16, - /// Columns of B / C. - pub n: u16, - /// Reduction dimension. - pub k: u16, - /// Element flavor. - pub dtype: MatMulType, - /// Accumulate into C instead of overwriting. - pub accumulate: bool, - /// Embedded (in-register) operand layout. - pub embedded: bool, - /// Optional quantization parameters. - pub quant: Option, - /// Optional operand signedness overrides. - pub input_signs: Option, -} - -/// Owned quantization parameters (mirrors [`MatMulQuant`](crate::function::MatMulQuant)). -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct QuantDesc { - /// Single shared scale, if any. - pub scale_scalar: Option, - /// Per-column scales, if any. - pub scale_per_column: Option>, - /// Apply ReLU to the output. - pub relu: bool, - /// Output flavor. - pub out: MatMulQuantOut, - /// Per-output bias, if any. - pub bias: Option>, - /// Zero point. - pub zero_point: i32, -} - -/// A `volt.matmul_config` attribute. -#[pliron_attr(name = "volt.matmul_config", verifier = "succ")] -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct MatMulAttr(pub MatMulDesc); - -impl MatMulAttr { - /// Wrap a descriptor. - pub fn new(desc: MatMulDesc) -> Self { - Self(desc) - } - - /// The descriptor. - pub fn desc(&self) -> &MatMulDesc { - &self.0 - } -} - -impl Printable for MatMulAttr { - fn fmt( - &self, - ctx: &Context, - state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let d = &self.0; - write!( - f, - "m={} n={} k={} dtype={} ", - d.m, - d.n, - d.k, - MatMulTypeAttr(d.dtype).print(ctx, state) - )?; - if d.accumulate { - write!(f, "acc ")?; - } - if d.embedded { - write!(f, "embedded ")?; - } - if let Some(q) = &d.quant { - write!(f, "quant(")?; - if let Some(s) = q.scale_scalar { - write!(f, "scale={s} ")?; - } - if let Some(scales) = &q.scale_per_column { - write!(f, "scales=[")?; - for (i, s) in scales.iter().enumerate() { - if i != 0 { - write!(f, ", ")?; - } - write!(f, "{s}")?; - } - write!(f, "] ")?; - } - if q.relu { - write!(f, "relu ")?; - } - write!(f, "out={} ", MatMulQuantOutAttr(q.out).print(ctx, state))?; - if let Some(bias) = &q.bias { - write!(f, "bias=[")?; - for (i, b) in bias.iter().enumerate() { - if i != 0 { - write!(f, ", ")?; - } - write!(f, "{b}")?; - } - write!(f, "] ")?; - } - write!(f, "zp={} ) ", q.zero_point)?; - } - if let Some(signs) = &d.input_signs { - write!( - f, - "signs(a_unsigned={} b_unsigned={}) ", - signs.a_unsigned, signs.b_unsigned - )?; - } - Ok(()) - } -} - -impl Parsable for MatMulAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - _state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - // TODO: picks up in the parser port. - unimplemented!("matmul config parsing") - } -} - -/// Call ABI details: variadic shape, struct-return, multi-register returns. -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct CallInfo { - /// Variadic call. - pub is_variadic: bool, - /// Number of fixed arguments (variadic calls). - pub num_fixed: u32, - /// Struct-return (first argument is the return slot). - pub sret: bool, - /// Number of registers used by the return value. - pub ret_regs: u8, - /// How the return value is split across registers. - pub ret_pieces: Vec, -} - -impl Default for CallInfo { - fn default() -> Self { - Self { - is_variadic: false, - num_fixed: 0, - sret: false, - ret_regs: 0, - ret_pieces: Vec::new(), - } - } -} - -/// A `volt.call_info` attribute. -#[pliron_attr(name = "volt.call_info", verifier = "succ")] -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct CallInfoAttr(pub CallInfo); - -impl CallInfoAttr { - /// Wrap call info. - pub fn new(info: CallInfo) -> Self { - Self(info) - } - - /// The call info. - pub fn info(&self) -> &CallInfo { - &self.0 - } -} - -impl Printable for CallInfoAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let c = &self.0; - write!( - f, - "variadic={} fixed={} sret={} regs={} pieces=[", - c.is_variadic, c.num_fixed, c.sret, c.ret_regs - )?; - for (i, p) in c.ret_pieces.iter().enumerate() { - if i != 0 { - write!(f, ", ")?; - } - write!(f, "{}:{}:{}", p.fp as u8, p.offset, p.bytes)?; - } - write!(f, "]") - } -} - -impl Parsable for CallInfoAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - _state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - // TODO: picks up in the parser port. - unimplemented!("call info parsing") - } -} - -/// Function-level flags: variadic shape, struct-return, linkage. -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)] -pub struct FuncInfo { - /// Variadic function. - pub is_variadic: bool, - /// Number of fixed parameters (variadic functions). - pub num_fixed_params: u32, - /// Struct-return (first parameter is the return slot). - pub sret: bool, - /// Local (non-exported) linkage. - pub is_local: bool, -} - -/// A `volt.func_info` attribute. -#[pliron_attr(name = "volt.func_info", verifier = "succ")] -#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Default)] -pub struct FuncInfoAttr(pub FuncInfo); - -impl FuncInfoAttr { - /// Wrap function info. - pub fn new(info: FuncInfo) -> Self { - Self(info) - } - - /// The function info. - pub fn info(&self) -> FuncInfo { - self.0 - } -} - -impl Printable for FuncInfoAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let i = &self.0; - write!( - f, - "variadic={} fixed={} sret={} local={}", - i.is_variadic, i.num_fixed_params, i.sret, i.is_local - ) - } -} - -impl Parsable for FuncInfoAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - _state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - // TODO: picks up in the parser port. - unimplemented!("func info parsing") - } -} - -/// A `volt.generic_attr` wrapping any legacy [`Attribute`], so existing -/// attribute flows (inline, align, endian, custom, ...) keep working. -/// Prints with the legacy syntax (`inline`, `align(16)`, `ns.key = 1`). -#[pliron_attr(name = "volt.generic_attr", verifier = "succ")] -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub struct VoltAttr(pub Attribute); - -impl VoltAttr { - /// Wrap a legacy attribute. - pub fn new(attr: Attribute) -> Self { - Self(attr) - } - - /// The wrapped attribute. - pub fn attr(&self) -> &Attribute { - &self.0 - } -} - -impl Printable for VoltAttr { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - match &self.0 { - Attribute::Inline => f.write_str("inline"), - Attribute::Noreturn => f.write_str("noreturn"), - Attribute::Cold => f.write_str("cold"), - Attribute::Align(a) => write!(f, "align({a})"), - Attribute::Endian(e) => { - let s = match e { - Endianness::Little => "little", - Endianness::Big => "big", - Endianness::Native => "native", - }; - write!(f, "endian({s})") - } - Attribute::Custom(c) => { - write!(f, "{}.{}", c.namespace, c.key)?; - match &c.value { - AttrValue::Flag => Ok(()), - AttrValue::Int(i) => write!(f, " = {i}"), - AttrValue::Str(s) => write!(f, " = \"{s}\""), - } - } - } - } -} - -fn quoted_string<'a>(state_stream: &mut StateStream<'a>) -> ParseResult<'a, String> { - token('"') - .with(pliron::combine::many(satisfy(|c: char| c != '"'))) - .skip(token('"')) - .map(|chars: Vec| chars.into_iter().collect::()) - .parse_stream(state_stream) - .into() -} - -impl Parsable for VoltAttr { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - _state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - // TODO: picks up in the parser port. - unimplemented!("generic attribute parsing") - } -} - -/// Parse a legacy dotted-path custom attribute name (`a.b.key`). -#[allow(dead_code)] -pub fn parse_custom_path<'a>(state_stream: &mut StateStream<'a>) -> ParseResult<'a, String> { - ( - pliron::combine::many1(satisfy(|c: char| c.is_ascii_alphanumeric() || c == '_')) - .map(|w: Vec| w.into_iter().collect::()), - pliron::combine::many(token('.').with(pliron::combine::many1(satisfy(|c: char| { - c.is_ascii_alphanumeric() || c == '_' - })))), - ) - .map(|(first, rest): (String, Vec>)| { - let mut path = first; - for seg in rest { - path.push('.'); - path.extend(seg); - } - path - }) - .parse_stream(state_stream) - .into() -} - -#[allow(dead_code)] -fn _use_quoted(state: &mut StateStream<'_>) { - let _ = quoted_string(state); - let _ = parse_custom_path(state); -} - -impl Custom { - /// Split a dotted path into namespace + key at the last dot. - #[allow(dead_code)] - pub fn split_path(path: String) -> Option { - let dot = path.rfind('.')?; - Some(Custom { - namespace: path[..dot].to_string(), - key: path[dot + 1..].to_string(), - value: AttrValue::Flag, - }) - } -} diff --git a/src/ir/src/dialect/func.rs b/src/ir/src/dialect/func.rs deleted file mode 100644 index 5e1daac..0000000 --- a/src/ir/src/dialect/func.rs +++ /dev/null @@ -1,1929 +0,0 @@ -//! Function-level container, builder, and legacy IR bridge over the `volt` dialect. - -extern crate alloc; - -use alloc::string::{String, ToString}; -use alloc::vec::Vec; -use core::fmt; -use std::collections::HashMap; - -use pliron::{ - basic_block::BasicBlock, - builtin::{ - op_interfaces::{OneRegionInterface, SymbolOpInterface}, - ops::FuncOp, - types::FunctionType, - }, - common_traits::Verify, - context::{Context, Ptr}, - identifier::Identifier, - linked_list::ContainsLinkedList, - op::Op, - operation::Operation, - printable::Printable, - region::Region, - r#type::{TypeHandle, Typed}, - value::Value, -}; - -use super::attrs::{CallInfo, FuncInfo, FuncInfoAttr, MatMulDesc, QuantDesc}; -use super::ops::{ - AllocaOp, ArithImmOp, ArithOp, AtomicRmwOp, BarrierOp, CallIndirectOp, CallOp, CondBrOp, - ConstF128Op, ConstFloatOp, ConstIntOp, ConvertOp, DecodeLowFloatOp, DequantizeNvfp4Op, DotOp, - EncodeLowFloatOp, ExtractOp, GlobalAddrOp, IcmpOp, JumpOp, LoadOp, MatMulOp, PrefetchOp, - QuantizeNvfp4Op, ReduceOp, RetOp, SelectOp, SplatOp, StoreOp, StructNewOp, UnaryOp, VaArgOp, - VaEndOp, VaStartOp, -}; -use super::types::{ - ArrayType, BoolType, FloatType, IntType, PtrType, SliceType, StructType, VectorType, VoltKind, - describe, -}; -use crate::function::{ - AtomicOp, AtomicOrdering, AtomicScope, BarrierScope, BinOp, CmpOp, RetPiece, -}; -use crate::low_float::Format as LowFloatFormat; -use crate::nvfp4::ScaleApplication; - -/// Error encountered during conversion between pliron IR and legacy IR. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ConvertError { - UnsupportedOp(String), - UnsupportedType(String), - MissingValue, - MissingBlock, - InvalidSignature, -} - -impl fmt::Display for ConvertError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::UnsupportedOp(op) => write!(f, "unsupported operation: {op}"), - Self::UnsupportedType(ty) => write!(f, "unsupported type: {ty}"), - Self::MissingValue => f.write_str("missing SSA value during lowering"), - Self::MissingBlock => f.write_str("missing block target during lowering"), - Self::InvalidSignature => f.write_str("invalid function signature"), - } - } -} - -impl core::error::Error for ConvertError {} - -/// A function container over the Volt dialect. -pub struct Function { - pub ctx: Context, - pub func: FuncOp, -} - -impl Function { - /// Create a new function with an owned context, entry block, and typed arguments. - pub fn new(name: &str, param_types: Vec, return_types: Vec) -> Self { - let mut ctx = Context::new(); - let func_ty = FunctionType::get(&mut ctx, param_types, return_types); - let id = Identifier::try_from(name).expect("valid function name"); - let func = FuncOp::new(&mut ctx, id, func_ty); - Self { ctx, func } - } - - /// Create a new function within an existing context. - pub fn with_context( - mut ctx: Context, - name: &str, - param_types: Vec, - return_types: Vec, - ) -> Self { - let func_ty = FunctionType::get(&mut ctx, param_types, return_types); - let id = Identifier::try_from(name).expect("valid function name"); - let func = FuncOp::new(&mut ctx, id, func_ty); - Self { ctx, func } - } - - /// Wrap an existing [`FuncOp`] and its context. - pub fn from_func_op(ctx: Context, func: FuncOp) -> Self { - Self { ctx, func } - } - - /// Reference to the underlying context. - pub fn context(&self) -> &Context { - &self.ctx - } - - /// Mutable reference to the underlying context. - pub fn context_mut(&mut self) -> &mut Context { - &mut self.ctx - } - - /// The function's [`FuncOp`]. - pub fn func_op(&self) -> FuncOp { - self.func - } - - /// The function symbol name. - pub fn name(&self) -> Identifier { - self.func.get_symbol_name(&self.ctx) - } - - /// The entry basic block. - pub fn entry_block(&self) -> Ptr { - self.func.get_entry_block(&self.ctx) - } - - /// The single region of the function. - pub fn region(&self) -> Ptr { - self.func.get_region(&self.ctx) - } - - /// All basic blocks in the function in order. - pub fn blocks(&self) -> Vec> { - self.region().deref(&self.ctx).iter(&self.ctx).collect() - } - - /// Append a new basic block to the function with optional label and arguments. - pub fn append_block( - &mut self, - label: Option<&str>, - arg_types: Vec, - ) -> Ptr { - let id = label.map(|s| Identifier::try_from(s).expect("valid block label")); - let bb = BasicBlock::new(&mut self.ctx, id, arg_types); - let r = self.region(); - bb.insert_at_back(r, &mut self.ctx); - bb - } - - /// Obtain an instruction builder positioned at the end of the given block. - pub fn builder(&mut self, block: Ptr) -> Builder<'_> { - Builder::new(&mut self.ctx, block) - } - - /// Set function-level information flags. - pub fn set_func_info(&mut self, info: FuncInfo) { - let key = pliron::ident!("volt_func_info"); - self.func - .get_operation() - .deref_mut(&mut self.ctx) - .attributes - .set(key, FuncInfoAttr::new(info)); - } - - /// Retrieve function-level information flags, if present. - pub fn func_info(&self) -> Option { - let key = pliron::ident!("volt_func_info"); - self.func - .get_operation() - .deref(&self.ctx) - .attributes - .get::(&key) - .map(|a| a.info()) - } - - /// Verify the function. - pub fn verify(&self) -> Result<(), pliron::result::Error> { - self.func.get_operation().deref(&self.ctx).verify(&self.ctx) - } - - /// Print the function to a string. - pub fn print(&self) -> String { - alloc::format!( - "{}", - self.func - .get_operation() - .deref(&self.ctx) - .print(&self.ctx, &Default::default()) - ) - } - - /// Convert a legacy [`crate::function::Function`] into a pliron [`Function`]. - pub fn from_legacy(legacy: &crate::function::Function) -> Self { - let mut ctx = Context::new(); - - // Convert all types up-front. - let type_map: Vec = (0..legacy.types.count()) - .map(|i| { - let ty = crate::types::Type(i as u32); - convert_legacy_type_to_pliron(&mut ctx, &legacy.types, ty) - }) - .collect(); - - // Determine input types and return types from the entry block and return statements. - let (param_types, ret_types) = if legacy.block_count() > 0 { - let entry_params = legacy.block_params(crate::function::Block(0)); - let params = entry_params - .iter() - .map(|&p| type_map[legacy.value_type(p).0 as usize]) - .collect(); - - // Find first return terminator to determine return types. - let mut rets = Vec::new(); - for b in 0..legacy.block_count() { - if let Some(crate::function::Terminator::Ret(r)) = - legacy.terminator(crate::function::Block(b as u32)) - { - rets = r - .slice() - .iter() - .map(|&v| type_map[legacy.value_type(v).0 as usize]) - .collect(); - break; - } - } - (params, rets) - } else { - (Vec::new(), Vec::new()) - }; - - let func_ty = FunctionType::get(&mut ctx, param_types, ret_types); - let func = FuncOp::new(&mut ctx, pliron::ident!("func"), func_ty); - let region = func.get_region(&ctx); - - if legacy.block_count() == 0 { - let mut res = Self { ctx, func }; - res.set_func_info(FuncInfo { - is_variadic: legacy.is_variadic, - num_fixed_params: legacy.num_fixed_params, - sret: legacy.sret, - is_local: legacy.is_local, - }); - return res; - } - - // Map blocks: block 0 is the entry block, blocks 1..N are created. - let mut blocks = Vec::with_capacity(legacy.block_count()); - let entry_bb = func.get_entry_block(&ctx); - blocks.push(entry_bb); - - for b in 1..legacy.block_count() { - let param_tys: Vec = legacy - .block_params(crate::function::Block(b as u32)) - .iter() - .map(|&p| type_map[legacy.value_type(p).0 as usize]) - .collect(); - let label = Identifier::try_from(alloc::format!("block{b}")).expect("valid block name"); - let bb = BasicBlock::new(&mut ctx, Some(label), param_tys); - bb.insert_at_back(region, &mut ctx); - blocks.push(bb); - } - - // Map values (indexed by legacy Value index). - let mut val_map: Vec> = alloc::vec![None; legacy.value_count()]; - - // Map entry block parameters. - for (i, &p) in legacy - .block_params(crate::function::Block(0)) - .iter() - .enumerate() - { - val_map[p.index()] = Some(entry_bb.deref(&ctx).get_argument(i)); - } - - // Map other block parameters. - for b in 1..legacy.block_count() { - let bb = blocks[b]; - for (i, &p) in legacy - .block_params(crate::function::Block(b as u32)) - .iter() - .enumerate() - { - val_map[p.index()] = Some(bb.deref(&ctx).get_argument(i)); - } - } - - // Translate instructions per block. - for b in 0..legacy.block_count() { - let bb = blocks[b]; - let legacy_b = crate::function::Block(b as u32); - - for inst in legacy.block_insts(legacy_b) { - let op = legacy.opcode(*inst); - let pliron_ty = legacy - .inst_result(*inst) - .map(|r| type_map[legacy.value_type(r).0 as usize]) - .unwrap_or_else(|| BoolType::get_type(&ctx)); - - match op { - crate::function::Opcode::Iconst(val) => { - let op = ConstIntOp::new(&mut ctx, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Fconst(val) => { - let op = ConstFloatOp::new(&mut ctx, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Fconst128(val) => { - let op = ConstF128Op::new(&mut ctx, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Arith(a) => { - let lhs = val_map[a.lhs.index()].expect("operand"); - let rhs = val_map[a.rhs.index()].expect("operand"); - let op = ArithOp::new(&mut ctx, a.op, lhs, rhs, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::ArithImm(a) => { - let lhs = val_map[a.lhs.index()].expect("operand"); - let op = ArithImmOp::new(&mut ctx, a.op, lhs, a.imm, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Icmp(c) => { - let lhs = val_map[c.lhs.index()].expect("operand"); - let rhs = val_map[c.rhs.index()].expect("operand"); - let op = IcmpOp::new(&mut ctx, c.op, lhs, rhs, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Select(s) => { - let cond = val_map[s.cond.index()].expect("operand"); - let t = val_map[s.then.index()].expect("operand"); - let e = val_map[s.else_.index()].expect("operand"); - let op = SelectOp::new(&mut ctx, cond, t, e, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::StructNew(s) => { - let fields: Vec<_> = legacy - .value_list(s.fields) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let op = StructNewOp::new(&mut ctx, fields, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Extract(e) => { - let agg = val_map[e.aggregate.index()].expect("operand"); - let op = ExtractOp::new(&mut ctx, agg, e.index, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Convert(c) => { - let val = val_map[c.value.index()].expect("operand"); - let op = ConvertOp::new(&mut ctx, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::DecodeLowFloat(d) => { - let val = val_map[d.value.index()].expect("operand"); - let op = DecodeLowFloatOp::new(&mut ctx, d.format, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::EncodeLowFloat(e) => { - let val = val_map[e.value.index()].expect("operand"); - let op = EncodeLowFloatOp::new(&mut ctx, e.format, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::DequantizeNvfp4(q) => { - let val = val_map[q.value.index()].expect("operand"); - let b_scale = val_map[q.block_scale.index()].expect("operand"); - let g_scale = val_map[q.global_scale.index()].expect("operand"); - let op = DequantizeNvfp4Op::new( - &mut ctx, - LowFloatFormat::F8E4M3, - val, - b_scale, - g_scale, - q.block_application, - q.global_application, - pliron_ty, - ); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::QuantizeNvfp4(q) => { - let val = val_map[q.value.index()].expect("operand"); - let b_scale = val_map[q.block_scale.index()].expect("operand"); - let g_scale = val_map[q.global_scale.index()].expect("operand"); - let op = QuantizeNvfp4Op::new( - &mut ctx, - LowFloatFormat::F8E4M3, - val, - b_scale, - g_scale, - q.block_application, - q.global_application, - pliron_ty, - ); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Unary(u) => { - let val = val_map[u.value.index()].expect("operand"); - let op = UnaryOp::new(&mut ctx, u.op, val, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Alloca(a) => { - let elem_ty = type_map[a.elem.0 as usize]; - let op = AllocaOp::new(&mut ctx, elem_ty, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Call(c) => { - let callee = legacy.symbol_name(c.symbol); - let args: Vec<_> = legacy - .value_list(c.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let res_tys = legacy - .inst_result(*inst) - .map(|_| alloc::vec![pliron_ty]) - .unwrap_or_default(); - let call_info = CallInfo { - is_variadic: c.is_variadic, - num_fixed: c.num_fixed, - sret: c.sret, - ret_regs: c.ret_regs, - ret_pieces: c.ret_pieces.to_vec(), - }; - let op = CallOp::new(&mut ctx, callee, args, res_tys, call_info); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::CallIndirect(c) => { - let target = val_map[c.target.index()].expect("operand"); - let args: Vec<_> = legacy - .value_list(c.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let res_tys = legacy - .inst_result(*inst) - .map(|_| alloc::vec![pliron_ty]) - .unwrap_or_default(); - let call_info = CallInfo { - is_variadic: c.is_variadic, - num_fixed: c.num_fixed, - sret: c.sret, - ret_regs: c.ret_regs, - ret_pieces: c.ret_pieces.to_vec(), - }; - let op = CallIndirectOp::new(&mut ctx, target, args, res_tys, call_info); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::GlobalAddr(g) => { - let symbol = legacy.symbol_name(g.symbol); - let op = GlobalAddrOp::new(&mut ctx, symbol, g.via_got, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Load(l) => { - let ptr = val_map[l.ptr.index()].expect("operand"); - let op = LoadOp::new(&mut ctx, ptr, l.volatile, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Store(s) => { - let ptr = val_map[s.ptr.index()].expect("operand"); - let val = val_map[s.value.index()].expect("operand"); - let op = StoreOp::new(&mut ctx, val, ptr, s.volatile); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::Prefetch(p) => { - let ptr = val_map[p.ptr.index()].expect("operand"); - let op = PrefetchOp::new(&mut ctx, ptr); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::VaStart(v) => { - let list = val_map[v.list.index()].expect("operand"); - let op = VaStartOp::new(&mut ctx, list); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::VaArg(v) => { - let list = val_map[v.list.index()].expect("operand"); - let op = VaArgOp::new(&mut ctx, list, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::VaEnd(v) => { - let list = val_map[v.list.index()].expect("operand"); - let op = VaEndOp::new(&mut ctx, list); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::Dot(d) => { - let acc = val_map[d.acc.index()].expect("operand"); - let a = val_map[d.a.index()].expect("operand"); - let b = val_map[d.b.index()].expect("operand"); - let op = DotOp::new(&mut ctx, acc, a, b, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Reduce(r) => { - let v = val_map[r.vector.index()].expect("operand"); - let op = ReduceOp::new(&mut ctx, r.op, v, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Splat(s) => { - let v = val_map[s.scalar.index()].expect("operand"); - let op = SplatOp::new(&mut ctx, v, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Matmul(m) => { - let a = val_map[m.a.index()].expect("operand"); - let b = val_map[m.b.index()].expect("operand"); - let c = val_map[m.c.index()].expect("operand"); - let desc = convert_matmul_desc_to_pliron(legacy, &m); - let op = MatMulOp::new(&mut ctx, a, b, c, desc, pliron_ty); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::Barrier(b_op) => { - let op = BarrierOp::new(&mut ctx, b_op.scope); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Opcode::AtomicRmw(a) => { - let ptr = val_map[a.ptr.index()].expect("operand"); - let val = val_map[a.value.index()].expect("operand"); - let cmp = a.compare.map(|c| val_map[c.index()].expect("operand")); - let op = AtomicRmwOp::new( - &mut ctx, a.op, ptr, val, cmp, a.ordering, a.scope, pliron_ty, - ); - op.get_operation().insert_at_back(bb, &mut ctx); - if let Some(r) = legacy.inst_result(*inst) { - val_map[r.index()] = Some(op.get_operation().deref(&ctx).get_result(0)); - } - } - crate::function::Opcode::If(cf) => { - let cond = val_map[cf.cond.index()].expect("operand"); - let then_dest = blocks[cf.then.target.index()]; - let then_args: Vec<_> = legacy - .value_list(cf.then.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let else_dest = blocks[cf.else_.target.index()]; - let else_args: Vec<_> = legacy - .value_list(cf.else_.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let op = CondBrOp::new( - &mut ctx, cond, then_dest, then_args, else_dest, else_args, - ); - op.get_operation().insert_at_back(bb, &mut ctx); - } - } - } - - // Emit terminator if present. - if let Some(term) = legacy.terminator(legacy_b) { - match term { - crate::function::Terminator::Ret(r) => { - let vals: Vec<_> = r - .slice() - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let op = RetOp::new(&mut ctx, vals); - op.get_operation().insert_at_back(bb, &mut ctx); - } - crate::function::Terminator::Jump(j) => { - let dest = blocks[j.target.index()]; - let args: Vec<_> = legacy - .value_list(j.args) - .iter() - .map(|&v| val_map[v.index()].expect("operand")) - .collect(); - let op = JumpOp::new(&mut ctx, dest, args); - op.get_operation().insert_at_back(bb, &mut ctx); - } - } - } - } - - let mut res = Self { ctx, func }; - res.set_func_info(FuncInfo { - is_variadic: legacy.is_variadic, - num_fixed_params: legacy.num_fixed_params, - sret: legacy.sret, - is_local: legacy.is_local, - }); - - res - } - - /// Convert this pliron [`Function`] into a legacy [`crate::function::Function`]. - pub fn to_legacy(&self) -> Result { - let mut legacy = crate::function::Function::new(); - - if let Some(info) = self.func_info() { - legacy.is_variadic = info.is_variadic; - legacy.num_fixed_params = info.num_fixed_params; - legacy.sret = info.sret; - legacy.is_local = info.is_local; - } - - let region = self.func.get_region(&self.ctx); - let blocks: Vec<_> = region.deref(&self.ctx).iter(&self.ctx).collect(); - if blocks.is_empty() { - return Ok(legacy); - } - - // Map pliron basic blocks to legacy Block indices. - let mut block_map: HashMap, crate::function::Block> = HashMap::new(); - let mut val_map: HashMap = HashMap::new(); - - for (i, &bb) in blocks.iter().enumerate() { - let legacy_b = legacy.append_block(); - assert_eq!(legacy_b.index(), i); - block_map.insert(bb, legacy_b); - } - - // Create block parameters. - for &bb in &blocks { - let legacy_b = block_map[&bb]; - let bb_ref = bb.deref(&self.ctx); - for arg_idx in 0..bb_ref.get_num_arguments() { - let arg = bb_ref.get_argument(arg_idx); - let ty = arg.get_type(&self.ctx); - let legacy_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let legacy_v = legacy.append_block_param(legacy_b, legacy_ty); - val_map.insert(arg, legacy_v); - } - } - - // Walk instructions in each block. - for &bb in &blocks { - let legacy_b = block_map[&bb]; - let bb_ref = bb.deref(&self.ctx); - - for op_ptr in bb_ref.iter(&self.ctx) { - let op = op_ptr.deref(&self.ctx); - let op_id = Operation::get_opid(op_ptr, &self.ctx); - let op_name = op_id.to_string(); - - match op_name.as_str() { - "volt.iconst" => { - let c = ConstIntOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Iconst(c.value(&self.ctx)), - ); - val_map.insert(op.get_result(0), v); - } - "volt.fconst" => { - let c = ConstFloatOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Fconst(c.value(&self.ctx)), - ); - val_map.insert(op.get_result(0), v); - } - "volt.fconst128" => { - let c = ConstF128Op::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Fconst128(c.value(&self.ctx)), - ); - val_map.insert(op.get_result(0), v); - } - "volt.arith" => { - let a = ArithOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let lhs = *val_map - .get(&a.lhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let rhs = *val_map - .get(&a.rhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Arith(crate::function::Arith { - op: a.kind(&self.ctx), - lhs, - rhs, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.arith_imm" => { - let a = ArithImmOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let lhs = *val_map - .get(&a.lhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::ArithImm(crate::function::ArithImm { - op: a.kind(&self.ctx), - lhs, - imm: a.imm(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.icmp" => { - let c = IcmpOp::from_operation(op_ptr); - let bool_ty = legacy.types.intern(crate::types::TypeKind::Bool); - let lhs = *val_map - .get(&c.lhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let rhs = *val_map - .get(&c.rhs(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - bool_ty, - crate::function::Opcode::Icmp(crate::function::Compare { - op: c.kind(&self.ctx), - lhs, - rhs, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.select" => { - let s = SelectOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (cond_v, then_raw, else_raw) = s.operands3(&self.ctx); - let cond = *val_map.get(&cond_v).ok_or(ConvertError::MissingValue)?; - let then_v = *val_map.get(&then_raw).ok_or(ConvertError::MissingValue)?; - let else_v = *val_map.get(&else_raw).ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Select(crate::function::Select { - cond, - then: then_v, - else_: else_v, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.struct_new" => { - let sn = StructNewOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let fields: Result, _> = sn - .fields(&self.ctx) - .iter() - .map(|f| val_map.get(f).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let v = legacy.append_struct_new(legacy_b, l_ty, &fields?); - val_map.insert(op.get_result(0), v); - } - "volt.extract" => { - let e = ExtractOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let agg = *val_map - .get(&e.aggregate(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Extract(crate::function::Extract { - aggregate: agg, - index: e.index(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.convert" => { - let c = ConvertOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let val = *val_map - .get(&c.value(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Convert(crate::function::Convert { - value: val, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.decode_low_float" => { - let d = DecodeLowFloatOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let val = *val_map - .get(&d.value(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::DecodeLowFloat( - crate::function::LowFloatConvert { - format: d.float_format(&self.ctx), - value: val, - }, - ), - ); - val_map.insert(op.get_result(0), v); - } - "volt.encode_low_float" => { - let e = EncodeLowFloatOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let val = *val_map - .get(&e.value(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::EncodeLowFloat( - crate::function::LowFloatConvert { - format: e.float_format(&self.ctx), - value: val, - }, - ), - ); - val_map.insert(op.get_result(0), v); - } - "volt.dequantize_nvfp4" => { - let q = DequantizeNvfp4Op::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (_fmt, b_app, g_app, v_raw, bs_raw, gs_raw) = q.parts(&self.ctx); - let val = *val_map.get(&v_raw).ok_or(ConvertError::MissingValue)?; - let b_scale = *val_map.get(&bs_raw).ok_or(ConvertError::MissingValue)?; - let g_scale = *val_map.get(&gs_raw).ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::DequantizeNvfp4( - crate::function::NvFp4Convert { - block_application: b_app, - global_application: g_app, - value: val, - block_scale: b_scale, - global_scale: g_scale, - }, - ), - ); - val_map.insert(op.get_result(0), v); - } - "volt.quantize_nvfp4" => { - let q = QuantizeNvfp4Op::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (_fmt, b_app, g_app, v_raw, bs_raw, gs_raw) = q.parts(&self.ctx); - let val = *val_map.get(&v_raw).ok_or(ConvertError::MissingValue)?; - let b_scale = *val_map.get(&bs_raw).ok_or(ConvertError::MissingValue)?; - let g_scale = *val_map.get(&gs_raw).ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::QuantizeNvfp4(crate::function::NvFp4Convert { - block_application: b_app, - global_application: g_app, - value: val, - block_scale: b_scale, - global_scale: g_scale, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.unary" => { - let u = UnaryOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let val = *val_map - .get(&u.value(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Unary(crate::function::Unary { - op: u.kind(&self.ctx), - value: val, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.alloca" => { - let a = AllocaOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let elem = convert_pliron_type_to_legacy( - &self.ctx, - &mut legacy.types, - a.elem_type(&self.ctx), - )?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Alloca(crate::function::Alloca { elem }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.call" => { - let c = CallOp::from_operation(op_ptr); - let callee = c.callee(&self.ctx); - let info = c.info(&self.ctx); - let args: Result, _> = c - .args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let args_list = legacy.intern_values(&args?); - let symbol = legacy.intern_symbol(&callee); - let mut ret_pieces = [RetPiece::default(); 4]; - for (dest, src) in ret_pieces.iter_mut().zip(&info.ret_pieces) { - *dest = *src; - } - let call = crate::function::Call { - symbol, - args: args_list, - is_variadic: info.is_variadic, - num_fixed: info.num_fixed, - ret_dest: None, - ret_regs: info.ret_regs, - ret_pieces, - sret: info.sret, - }; - if op.get_num_results() > 0 { - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = - convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Call(call), - ); - val_map.insert(op.get_result(0), v); - } else { - legacy.append_stmt_raw(legacy_b, crate::function::Opcode::Call(call)); - } - } - "volt.call_indirect" => { - let c = CallIndirectOp::from_operation(op_ptr); - let target = *val_map - .get(&c.target(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let info = c.info(&self.ctx); - let args: Result, _> = c - .args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let args_list = legacy.intern_values(&args?); - let mut ret_pieces = [RetPiece::default(); 4]; - for (dest, src) in ret_pieces.iter_mut().zip(&info.ret_pieces) { - *dest = *src; - } - let call = crate::function::CallIndirect { - target, - args: args_list, - is_variadic: info.is_variadic, - num_fixed: info.num_fixed, - ret_dest: None, - ret_regs: info.ret_regs, - ret_pieces, - sret: info.sret, - }; - if op.get_num_results() > 0 { - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = - convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::CallIndirect(call), - ); - val_map.insert(op.get_result(0), v); - } else { - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::CallIndirect(call), - ); - } - } - "volt.global_addr" => { - let g = GlobalAddrOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let symbol = legacy.intern_symbol(&g.symbol(&self.ctx)); - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::GlobalAddr(crate::function::GlobalAddr { - symbol, - via_got: g.via_got(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.load" => { - let l = LoadOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let ptr = *val_map - .get(&l.ptr(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Load(crate::function::Load { - ptr, - volatile: l.volatile(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.store" => { - let s = StoreOp::from_operation(op_ptr); - let (v_raw, p_raw) = s.parts(&self.ctx); - let val = *val_map.get(&v_raw).ok_or(ConvertError::MissingValue)?; - let ptr = *val_map.get(&p_raw).ok_or(ConvertError::MissingValue)?; - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::Store(crate::function::Store { - ptr, - value: val, - volatile: s.volatile(&self.ctx), - }), - ); - } - "volt.prefetch" => { - let p = PrefetchOp::from_operation(op_ptr); - let ptr = *val_map - .get(&p.ptr(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::Prefetch(crate::function::Prefetch { ptr }), - ); - } - "volt.va_start" => { - let v = VaStartOp::from_operation(op_ptr); - let list = *val_map - .get(&v.list(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::VaStart(crate::function::VaStart { list }), - ); - } - "volt.va_arg" => { - let v = VaArgOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let list = *val_map - .get(&v.list(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let val = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::VaArg(crate::function::VaArg { - list, - ty: l_ty, - }), - ); - val_map.insert(op.get_result(0), val); - } - "volt.va_end" => { - let v = VaEndOp::from_operation(op_ptr); - let list = *val_map - .get(&v.list(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::VaEnd(crate::function::VaEnd { list }), - ); - } - "volt.dot" => { - let d = DotOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (acc_raw, a_raw, b_raw) = d.operands3(&self.ctx); - let acc = *val_map.get(&acc_raw).ok_or(ConvertError::MissingValue)?; - let a = *val_map.get(&a_raw).ok_or(ConvertError::MissingValue)?; - let b = *val_map.get(&b_raw).ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Dot(crate::function::Dot { acc, a, b }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.reduce" => { - let r = ReduceOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let vector = *val_map - .get(&r.vector(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Reduce(crate::function::Reduce { - op: r.kind(&self.ctx), - vector, - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.splat" => { - let s = SplatOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let scalar = *val_map - .get(&s.scalar(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::Splat(crate::function::Splat { scalar }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.matmul" => { - let m = MatMulOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (a_raw, b_raw, c_raw) = m.operands3(&self.ctx); - let a = *val_map.get(&a_raw).ok_or(ConvertError::MissingValue)?; - let b = *val_map.get(&b_raw).ok_or(ConvertError::MissingValue)?; - let c = *val_map.get(&c_raw).ok_or(ConvertError::MissingValue)?; - let desc = m.config(&self.ctx); - let mm = convert_matmul_desc_to_legacy(&mut legacy, &desc, a, b, c); - let v = - legacy.append_inst(legacy_b, l_ty, crate::function::Opcode::Matmul(mm)); - val_map.insert(op.get_result(0), v); - } - "volt.barrier" => { - let b_op = BarrierOp::from_operation(op_ptr); - legacy.append_stmt_raw( - legacy_b, - crate::function::Opcode::Barrier(crate::function::Barrier { - scope: b_op.scope(&self.ctx), - }), - ); - } - "volt.atomic_rmw" => { - let a = AtomicRmwOp::from_operation(op_ptr); - let ty = op.get_result(0).get_type(&self.ctx); - let l_ty = convert_pliron_type_to_legacy(&self.ctx, &mut legacy.types, ty)?; - let (p_raw, v_raw, cmp_raw) = a.operands_rmw(&self.ctx); - let ptr = *val_map.get(&p_raw).ok_or(ConvertError::MissingValue)?; - let value = *val_map.get(&v_raw).ok_or(ConvertError::MissingValue)?; - let compare = cmp_raw - .map(|c| val_map.get(&c).copied().ok_or(ConvertError::MissingValue)) - .transpose()?; - let v = legacy.append_inst( - legacy_b, - l_ty, - crate::function::Opcode::AtomicRmw(crate::function::AtomicRmw { - op: a.kind(&self.ctx), - ptr, - value, - compare, - ordering: a.ordering(&self.ctx), - scope: a.scope(&self.ctx), - }), - ); - val_map.insert(op.get_result(0), v); - } - "volt.cond_br" => { - let br = CondBrOp::from_operation(op_ptr); - let cond = *val_map - .get(&br.cond(&self.ctx)) - .ok_or(ConvertError::MissingValue)?; - let then_target = *block_map - .get(&br.then_dest(&self.ctx)) - .ok_or(ConvertError::MissingBlock)?; - let then_args: Result, _> = br - .then_args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let else_target = *block_map - .get(&br.else_dest(&self.ctx)) - .ok_or(ConvertError::MissingBlock)?; - let else_args: Result, _> = br - .else_args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let then_edge = crate::function::EdgeDesc { - target: then_target, - args: then_args?, - }; - let else_edge = crate::function::EdgeDesc { - target: else_target, - args: else_args?, - }; - legacy.append_if(legacy_b, cond, then_edge, else_edge); - } - "volt.jump" => { - let j = JumpOp::from_operation(op_ptr); - let target = *block_map - .get(&j.target(&self.ctx)) - .ok_or(ConvertError::MissingBlock)?; - let args: Vec<_> = j - .args(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect::, _>>()?; - let interned = legacy.intern_values(&args); - legacy.set_terminator( - legacy_b, - crate::function::Terminator::Jump(crate::function::Jump { - target, - args: interned, - }), - ); - } - "volt.ret" => { - let r = RetOp::from_operation(op_ptr); - let vals: Result, _> = r - .values(&self.ctx) - .iter() - .map(|a| val_map.get(a).copied().ok_or(ConvertError::MissingValue)) - .collect(); - let vals = vals?; - legacy.set_terminator( - legacy_b, - crate::function::Terminator::Ret(crate::function::Ret::many(&vals)), - ); - } - _ => return Err(ConvertError::UnsupportedOp(op_name)), - } - } - } - - Ok(legacy) - } -} - -/// An instruction builder for pliron functions. -pub struct Builder<'a> { - pub ctx: &'a mut Context, - pub block: Ptr, -} - -impl<'a> Builder<'a> { - /// Create a new builder in the given block. - pub fn new(ctx: &'a mut Context, block: Ptr) -> Self { - Self { ctx, block } - } - - /// Reposition the builder at the end of another block. - pub fn switch_to(&mut self, block: Ptr) { - self.block = block; - } - - /// The current block being appended to. - pub fn block(&self) -> Ptr { - self.block - } - - /// Emit an integer constant. - pub fn iconst(&mut self, val: i64, ty: TypeHandle) -> Value { - let op = ConstIntOp::new(self.ctx, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a double-precision float constant. - pub fn fconst(&mut self, val: f64, ty: TypeHandle) -> Value { - let op = ConstFloatOp::new(self.ctx, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit an f128 constant. - pub fn fconst128(&mut self, bits: u128, ty: TypeHandle) -> Value { - let op = ConstF128Op::new(self.ctx, bits, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit binary arithmetic. - pub fn arith(&mut self, kind: BinOp, lhs: Value, rhs: Value, ty: TypeHandle) -> Value { - let op = ArithOp::new(self.ctx, kind, lhs, rhs, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit binary arithmetic with an immediate. - pub fn arith_imm(&mut self, kind: BinOp, lhs: Value, imm: i64, ty: TypeHandle) -> Value { - let op = ArithImmOp::new(self.ctx, kind, lhs, imm, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit an integer comparison. - pub fn icmp(&mut self, kind: CmpOp, lhs: Value, rhs: Value) -> Value { - let bool_ty = BoolType::get_type(self.ctx); - let op = IcmpOp::new(self.ctx, kind, lhs, rhs, bool_ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a select. - pub fn select( - &mut self, - cond: Value, - then_val: Value, - else_val: Value, - ty: TypeHandle, - ) -> Value { - let op = SelectOp::new(self.ctx, cond, then_val, else_val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit struct construction. - pub fn struct_new(&mut self, fields: Vec, ty: TypeHandle) -> Value { - let op = StructNewOp::new(self.ctx, fields, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit struct field extraction. - pub fn extract(&mut self, aggregate: Value, index: u32, ty: TypeHandle) -> Value { - let op = ExtractOp::new(self.ctx, aggregate, index, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit type conversion. - pub fn convert(&mut self, val: Value, ty: TypeHandle) -> Value { - let op = ConvertOp::new(self.ctx, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit low-float decode. - pub fn decode_low_float(&mut self, fmt: LowFloatFormat, val: Value, ty: TypeHandle) -> Value { - let op = DecodeLowFloatOp::new(self.ctx, fmt, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit low-float encode. - pub fn encode_low_float(&mut self, fmt: LowFloatFormat, val: Value, ty: TypeHandle) -> Value { - let op = EncodeLowFloatOp::new(self.ctx, fmt, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit NVFP4 dequantize. - pub fn dequantize_nvfp4( - &mut self, - fmt: LowFloatFormat, - val: Value, - b_scale: Value, - g_scale: Value, - b_app: ScaleApplication, - g_app: ScaleApplication, - ty: TypeHandle, - ) -> Value { - let op = DequantizeNvfp4Op::new(self.ctx, fmt, val, b_scale, g_scale, b_app, g_app, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit NVFP4 quantize. - pub fn quantize_nvfp4( - &mut self, - fmt: LowFloatFormat, - val: Value, - b_scale: Value, - g_scale: Value, - b_app: ScaleApplication, - g_app: ScaleApplication, - ty: TypeHandle, - ) -> Value { - let op = QuantizeNvfp4Op::new(self.ctx, fmt, val, b_scale, g_scale, b_app, g_app, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a unary operation. - pub fn unary(&mut self, kind: crate::function::UnaryOp, val: Value, ty: TypeHandle) -> Value { - let op = UnaryOp::new(self.ctx, kind, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit alloca. - pub fn alloca(&mut self, elem_ty: TypeHandle, ptr_ty: TypeHandle) -> Value { - let op = AllocaOp::new(self.ctx, elem_ty, ptr_ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a direct call. - pub fn call( - &mut self, - callee: &str, - args: Vec, - res_tys: Vec, - info: CallInfo, - ) -> Vec { - let op = CallOp::new(self.ctx, callee, args, res_tys, info); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).results().collect() - } - - /// Emit an indirect call. - pub fn call_indirect( - &mut self, - target: Value, - args: Vec, - res_tys: Vec, - info: CallInfo, - ) -> Vec { - let op = CallIndirectOp::new(self.ctx, target, args, res_tys, info); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).results().collect() - } - - /// Emit a global address reference. - pub fn global_addr(&mut self, symbol: &str, via_got: bool, ty: TypeHandle) -> Value { - let op = GlobalAddrOp::new(self.ctx, symbol, via_got, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a load. - pub fn load(&mut self, ptr: Value, volatile: bool, ty: TypeHandle) -> Value { - let op = LoadOp::new(self.ctx, ptr, volatile, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a store. - pub fn store(&mut self, ptr: Value, val: Value, volatile: bool) { - let op = StoreOp::new(self.ctx, val, ptr, volatile); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit prefetch. - pub fn prefetch(&mut self, ptr: Value) { - let op = PrefetchOp::new(self.ctx, ptr); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit va_start. - pub fn va_start(&mut self, list: Value) { - let op = VaStartOp::new(self.ctx, list); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit va_arg. - pub fn va_arg(&mut self, list: Value, ty: TypeHandle) -> Value { - let op = VaArgOp::new(self.ctx, list, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit va_end. - pub fn va_end(&mut self, list: Value) { - let op = VaEndOp::new(self.ctx, list); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit dot product. - pub fn dot(&mut self, acc: Value, a: Value, b: Value, ty: TypeHandle) -> Value { - let op = DotOp::new(self.ctx, acc, a, b, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit vector reduction. - pub fn reduce(&mut self, kind: BinOp, vec: Value, ty: TypeHandle) -> Value { - let op = ReduceOp::new(self.ctx, kind, vec, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit vector splat. - pub fn splat(&mut self, val: Value, ty: TypeHandle) -> Value { - let op = SplatOp::new(self.ctx, val, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit matrix multiplication. - pub fn matmul( - &mut self, - a: Value, - b: Value, - c: Value, - desc: MatMulDesc, - ty: TypeHandle, - ) -> Value { - let op = MatMulOp::new(self.ctx, a, b, c, desc, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit a barrier. - pub fn barrier(&mut self, scope: BarrierScope) { - let op = BarrierOp::new(self.ctx, scope); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit an atomic RMW operation. - pub fn atomic_rmw( - &mut self, - op: AtomicOp, - ptr: Value, - val: Value, - cmp: Option, - ordering: AtomicOrdering, - scope: AtomicScope, - ty: TypeHandle, - ) -> Value { - let op = AtomicRmwOp::new(self.ctx, op, ptr, val, cmp, ordering, scope, ty); - op.get_operation().insert_at_back(self.block, self.ctx); - op.get_operation().deref(self.ctx).get_result(0) - } - - /// Emit an unconditional jump. - pub fn jump(&mut self, target: Ptr, args: Vec) { - let op = JumpOp::new(self.ctx, target, args); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit a conditional branch. - pub fn cond_br( - &mut self, - cond: Value, - then_dest: Ptr, - then_args: Vec, - else_dest: Ptr, - else_args: Vec, - ) { - let op = CondBrOp::new(self.ctx, cond, then_dest, then_args, else_dest, else_args); - op.get_operation().insert_at_back(self.block, self.ctx); - } - - /// Emit a return. - pub fn ret(&mut self, values: Vec) { - let op = RetOp::new(self.ctx, values); - op.get_operation().insert_at_back(self.block, self.ctx); - } -} - -fn convert_legacy_type_to_pliron( - ctx: &mut Context, - types: &crate::types::TypeTable, - ty: crate::types::Type, -) -> TypeHandle { - match types.type_kind(ty) { - crate::types::TypeKind::Bool => BoolType::get_type(ctx), - crate::types::TypeKind::Int(desc) => IntType::get_type(ctx, desc.signed, desc.bits as u32), - crate::types::TypeKind::Float(k) => FloatType::get_type(ctx, *k), - crate::types::TypeKind::Ptr(s) => PtrType::get_type(ctx, *s), - crate::types::TypeKind::Vector(v) => { - let elem = convert_legacy_type_to_pliron(ctx, types, v.elem); - VectorType::get_type(ctx, v.len, elem) - } - crate::types::TypeKind::Array(a) => { - let elem = convert_legacy_type_to_pliron(ctx, types, a.elem); - ArrayType::get_type(ctx, a.len, elem) - } - crate::types::TypeKind::Slice(s) => { - let elem = convert_legacy_type_to_pliron(ctx, types, s.elem); - SliceType::get_type(ctx, elem) - } - crate::types::TypeKind::Struct(fields) => { - let field_tys: Vec<_> = fields - .iter() - .map(|&f| convert_legacy_type_to_pliron(ctx, types, f)) - .collect(); - StructType::get_type(ctx, field_tys) - } - } -} - -fn convert_pliron_type_to_legacy( - ctx: &Context, - types: &mut crate::types::TypeTable, - ty: TypeHandle, -) -> Result { - let kind = describe(ctx, ty).ok_or_else(|| { - ConvertError::UnsupportedType(alloc::format!( - "{}", - ty.deref(ctx).print(ctx, &Default::default()) - )) - })?; - - let l_kind = match kind { - VoltKind::Bool => crate::types::TypeKind::Bool, - VoltKind::Int { signed, bits } => crate::types::TypeKind::Int(crate::types::IntDesc { - signed, - bits: bits as u16, - }), - VoltKind::Float(k) => crate::types::TypeKind::Float(k), - VoltKind::Ptr(s) => crate::types::TypeKind::Ptr(s), - VoltKind::Vector { len, elem } => { - let e = convert_pliron_type_to_legacy(ctx, types, elem)?; - crate::types::TypeKind::Vector(crate::types::VectorDesc { len, elem: e }) - } - VoltKind::Array { len, elem } => { - let e = convert_pliron_type_to_legacy(ctx, types, elem)?; - crate::types::TypeKind::Array(crate::types::ArrayDesc { len, elem: e }) - } - VoltKind::Slice { elem } => { - let e = convert_pliron_type_to_legacy(ctx, types, elem)?; - crate::types::TypeKind::Slice(crate::types::SliceDesc { elem: e }) - } - VoltKind::Struct(fields) => { - let mut l_fields = Vec::with_capacity(fields.len()); - for f in fields { - l_fields.push(convert_pliron_type_to_legacy(ctx, types, f)?); - } - crate::types::TypeKind::Struct(l_fields) - } - }; - Ok(types.intern(l_kind)) -} - -fn convert_matmul_desc_to_pliron( - legacy: &crate::function::Function, - m: &crate::function::MatMul, -) -> MatMulDesc { - MatMulDesc { - m: m.m, - n: m.n, - k: m.k, - dtype: m.dtype, - accumulate: m.accumulate, - embedded: m.embedded, - quant: m.quant.as_ref().map(|q| { - let (scalar, col) = match q.scale { - crate::function::MatMulScale::Scalar(s) => (Some(s), None), - crate::function::MatMulScale::PerColumn(cl) => { - (None, Some(legacy.scale_list(cl).to_vec())) - } - }; - QuantDesc { - scale_scalar: scalar, - scale_per_column: col, - relu: q.relu, - out: q.out, - bias: q.bias.map(|b| legacy.bias_list(b).to_vec()), - zero_point: q.zero_point, - } - }), - input_signs: m.input_signs, - } -} - -fn convert_matmul_desc_to_legacy( - legacy: &mut crate::function::Function, - desc: &MatMulDesc, - a: crate::function::Value, - b: crate::function::Value, - c: crate::function::Value, -) -> crate::function::MatMul { - let quant = desc.quant.as_ref().map(|q| { - let scale = if let Some(s) = q.scale_scalar { - crate::function::MatMulScale::Scalar(s) - } else if let Some(cols) = &q.scale_per_column { - crate::function::MatMulScale::PerColumn(legacy.intern_scales(cols)) - } else { - crate::function::MatMulScale::Scalar(0) - }; - let bias = q.bias.as_ref().map(|biases| legacy.intern_bias(biases)); - crate::function::MatMulQuant { - scale, - relu: q.relu, - out: q.out, - bias, - zero_point: q.zero_point, - } - }); - - crate::function::MatMul { - a, - b, - c, - m: desc.m, - n: desc.n, - k: desc.k, - dtype: desc.dtype, - accumulate: desc.accumulate, - embedded: desc.embedded, - quant, - input_signs: desc.input_signs, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn builder_emits_valid_function() { - let mut f = Function::new("test_fn", vec![], vec![]); - let i32t = IntType::get_type(&f.ctx, true, 32); - let entry = f.entry_block(); - let mut b = f.builder(entry); - let v1 = b.iconst(10, i32t); - let v2 = b.iconst(20, i32t); - let sum = b.arith(BinOp::Add, v1, v2, i32t); - b.ret(vec![sum]); - - let text = f.print(); - assert!(text.contains("volt.iconst 10"), "{text}"); - assert!(text.contains("volt.iconst 20"), "{text}"); - assert!(text.contains("volt.arith add"), "{text}"); - assert!(text.contains("volt.ret"), "{text}"); - assert!(f.verify().is_ok()); - } - - #[test] - fn round_trip_legacy_function() { - let mut legacy = crate::function::Function::new(); - let i32_t = legacy - .types - .intern(crate::types::TypeKind::Int(crate::types::IntDesc { - signed: true, - bits: 32, - })); - let b0 = legacy.append_block(); - let p0 = legacy.append_block_param(b0, i32_t); - let c1 = legacy.append_inst(b0, i32_t, crate::function::Opcode::Iconst(42)); - let add = legacy.append_inst( - b0, - i32_t, - crate::function::Opcode::Arith(crate::function::Arith { - op: BinOp::Add, - lhs: p0, - rhs: c1, - }), - ); - legacy.set_terminator( - b0, - crate::function::Terminator::Ret(crate::function::Ret::one(add)), - ); - - let pliron_f = Function::from_legacy(&legacy); - assert!(pliron_f.verify().is_ok()); - - let round_tripped = pliron_f.to_legacy().unwrap(); - assert_eq!(round_tripped.block_count(), legacy.block_count()); - assert_eq!(round_tripped.block_params(b0).len(), 1); - assert_eq!(round_tripped.block_insts(b0).len(), 2); - } - - #[test] - fn round_trip_control_flow() { - let mut legacy = crate::function::Function::new(); - let bool_t = legacy.types.intern(crate::types::TypeKind::Bool); - let i32_t = legacy - .types - .intern(crate::types::TypeKind::Int(crate::types::IntDesc { - signed: true, - bits: 32, - })); - let b0 = legacy.append_block(); - let cond = legacy.append_block_param(b0, bool_t); - let b1 = legacy.append_block(); - let p1 = legacy.append_block_param(b1, i32_t); - let b2 = legacy.append_block(); - let p2 = legacy.append_block_param(b2, i32_t); - - let c1 = legacy.append_inst(b0, i32_t, crate::function::Opcode::Iconst(1)); - let c2 = legacy.append_inst(b0, i32_t, crate::function::Opcode::Iconst(2)); - - let then_edge = crate::function::EdgeDesc { - target: b1, - args: vec![c1], - }; - let else_edge = crate::function::EdgeDesc { - target: b2, - args: vec![c2], - }; - legacy.append_if(b0, cond, then_edge, else_edge); - legacy.set_terminator( - b1, - crate::function::Terminator::Ret(crate::function::Ret::one(p1)), - ); - legacy.set_terminator( - b2, - crate::function::Terminator::Ret(crate::function::Ret::one(p2)), - ); - - let pliron_f = Function::from_legacy(&legacy); - assert!(pliron_f.verify().is_ok()); - - let round_tripped = pliron_f.to_legacy().unwrap(); - assert_eq!(round_tripped.block_count(), 3); - assert_eq!(round_tripped.block_params(b1).len(), 1); - assert_eq!(round_tripped.block_params(b2).len(), 1); - } - - #[test] - fn round_trip_memory_ops() { - let mut legacy = crate::function::Function::new(); - let i32_t = legacy - .types - .intern(crate::types::TypeKind::Int(crate::types::IntDesc { - signed: true, - bits: 32, - })); - let ptr_t = legacy.types.intern(crate::types::TypeKind::Ptr( - crate::types::AddressSpace::Global, - )); - let b0 = legacy.append_block(); - let slot = legacy.append_inst( - b0, - ptr_t, - crate::function::Opcode::Alloca(crate::function::Alloca { elem: i32_t }), - ); - let c42 = legacy.append_inst(b0, i32_t, crate::function::Opcode::Iconst(42)); - legacy.append_store(b0, c42, slot); - let loaded = legacy.append_inst( - b0, - i32_t, - crate::function::Opcode::Load(crate::function::Load { - ptr: slot, - volatile: false, - }), - ); - legacy.set_terminator( - b0, - crate::function::Terminator::Ret(crate::function::Ret::one(loaded)), - ); - - let pliron_f = Function::from_legacy(&legacy); - assert!(pliron_f.verify().is_ok()); - - let round_tripped = pliron_f.to_legacy().unwrap(); - assert_eq!(round_tripped.block_count(), 1); - assert_eq!(round_tripped.block_insts(b0).len(), 4); - } -} diff --git a/src/ir/src/dialect/mod.rs b/src/ir/src/dialect/mod.rs deleted file mode 100644 index a4ab5c3..0000000 --- a/src/ir/src/dialect/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -//! The `volt` dialect: volt IR modeled with pliron. -//! -//! [`types`] defines the volt type system as pliron types, [`attrs`] -//! defines operation payload attributes, [`ops`] defines every volt -//! operation, and [`func`] provides the function-level container and -//! builder used by passes, targets, and the emulator. - -pub mod attrs; -pub mod func; -pub mod ops; -pub mod types; - -/// The volt dialect name, used as the `volt.` prefix on ops, types, -/// and attributes. -pub const DIALECT_NAME: &str = "volt"; diff --git a/src/ir/src/dialect/ops.rs b/src/ir/src/dialect/ops.rs deleted file mode 100644 index 381a707..0000000 --- a/src/ir/src/dialect/ops.rs +++ /dev/null @@ -1,1674 +0,0 @@ -//! Every `volt` dialect operation. -//! -//! The op set mirrors the legacy [`Opcode`](crate::function::Opcode): -//! constants, arithmetic, control flow, memory, calls, vector/matrix and -//! atomics. Most ops use a declarative assembly format; [`CondBrOp`] -//! (two successors with partitioned operands) has a manual -//! printer/parser. - -use alloc::string::String; -use alloc::vec::Vec; - -use pliron::{ - builtin::{ - attributes::{BoolAttr, StringAttr, TypeAttr}, - op_interfaces::{ - IsTerminatorInterface, NOpdsInterface, NResultsInterface, OneResultInterface, - }, - }, - combine::Parser, - combine::parser::token::token, - common_traits::Named, - context::{Context, Ptr}, - derive::pliron_op, - identifier::Identifier, - input_err, - irfmt::parsers::{ - block_opd_parser, delimited_list_parser, process_parsed_ssa_defs, spaced, ssa_opd_parser, - }, - location::{Located, Location}, - op::{Op, OpObj}, - operation::Operation, - parsable::{IntoParseResult, Parsable, ParseResult, StateStream}, - printable::{self, Printable}, - r#type::TypeHandle, - value::Value, -}; - -use super::super::function::{AtomicOp, AtomicOrdering, AtomicScope, BarrierScope, BinOp, CmpOp}; -use super::super::low_float::Format as LowFloatFormat; -use super::super::nvfp4::ScaleApplication; -use super::attrs::{ - AtomicOpAttr, AtomicOrderingAttr, AtomicScopeAttr, BarrierAttr, BinOpAttr, CallInfo, - CallInfoAttr, CmpOpAttr, F128BitsAttr, FloatBitsAttr, ImmAttr, LowFloatAttr, MatMulAttr, - MatMulDesc, ScaleAppAttr, UnaryOpAttr, -}; -use pliron::basic_block::BasicBlock; - -// Fetch a required attribute, panicking with the op name on absence. -// Attributes are always set by constructors; absence is a bug. -fn req_attr(opt: Option>, op: &str, attr: &str) -> T { - opt.map(|r| T::clone(&r)) - .unwrap_or_else(|| panic!("{op} is missing required attribute {attr}")) -} - -/// An integer constant: `%r = volt.iconst 42 : volt.int`. -#[pliron_op( - name = "volt.iconst", - format = "attr($iconst_value, $ImmAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (iconst_value: ImmAttr), - verifier = "succ", -)] -pub struct ConstIntOp; - -impl ConstIntOp { - /// Build an integer constant. - pub fn new(ctx: &mut Context, value: i64, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_iconst_value(ctx, ImmAttr::new(value)); - op - } - - /// The constant value. - pub fn value(&self, ctx: &Context) -> i64 { - req_attr( - self.get_attr_iconst_value(ctx), - "volt.iconst", - "iconst_value", - ) - .value() - } -} - -/// A double-precision constant (stored as raw bits). -#[pliron_op( - name = "volt.fconst", - format = "attr($fconst_value, $FloatBitsAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (fconst_value: FloatBitsAttr), - verifier = "succ", -)] -pub struct ConstFloatOp; - -impl ConstFloatOp { - /// Build a float constant. - pub fn new(ctx: &mut Context, value: f64, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_fconst_value(ctx, FloatBitsAttr::new(value)); - op - } - - /// The constant value. - pub fn value(&self, ctx: &Context) -> f64 { - req_attr( - self.get_attr_fconst_value(ctx), - "volt.fconst", - "fconst_value", - ) - .value() - } -} - -/// A quad-precision constant (stored as raw bits). -#[pliron_op( - name = "volt.fconst128", - format = "attr($fconst128_value, $F128BitsAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (fconst128_value: F128BitsAttr), - verifier = "succ", -)] -pub struct ConstF128Op; - -impl ConstF128Op { - /// Build an f128 constant. - pub fn new(ctx: &mut Context, value: u128, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_fconst128_value(ctx, F128BitsAttr::new(value)); - op - } - - /// The constant value. - pub fn value(&self, ctx: &Context) -> u128 { - req_attr( - self.get_attr_fconst128_value(ctx), - "volt.fconst128", - "fconst128_value", - ) - .value() - } -} - -/// Binary arithmetic: `volt.arith add %lhs, %rhs : T`. -#[pliron_op( - name = "volt.arith", - format = "attr($arith_kind, $BinOpAttr) ` ` $0 `, ` $1 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (arith_kind: BinOpAttr), - verifier = "succ", -)] -pub struct ArithOp; - -impl ArithOp { - /// Build a binary arithmetic op. - pub fn new(ctx: &mut Context, kind: BinOp, lhs: Value, rhs: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![lhs, rhs], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_arith_kind(ctx, BinOpAttr::new(kind)); - op - } - - /// The arithmetic operator. - pub fn kind(&self, ctx: &Context) -> BinOp { - req_attr(self.get_attr_arith_kind(ctx), "volt.arith", "arith_kind").value() - } - - /// Left operand. - pub fn lhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Right operand. - pub fn rhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(1) - } -} - -/// Arithmetic with an immediate: `volt.arith_imm add %lhs, 5 : T`. -#[pliron_op( - name = "volt.arith_imm", - format = "attr($arith_imm_kind, $BinOpAttr) ` ` $0 `, ` attr($arith_imm_imm, $ImmAttr) ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (arith_imm_kind: BinOpAttr, arith_imm_imm: ImmAttr), - verifier = "succ", -)] -pub struct ArithImmOp; - -impl ArithImmOp { - /// Build an arithmetic-immediate op. - pub fn new(ctx: &mut Context, kind: BinOp, lhs: Value, imm: i64, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![lhs], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_arith_imm_kind(ctx, BinOpAttr::new(kind)); - op.set_attr_arith_imm_imm(ctx, ImmAttr::new(imm)); - op - } - - /// The arithmetic operator. - pub fn kind(&self, ctx: &Context) -> BinOp { - req_attr( - self.get_attr_arith_imm_kind(ctx), - "volt.arith_imm", - "arith_imm_kind", - ) - .value() - } - - /// The value operand. - pub fn lhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// The immediate. - pub fn imm(&self, ctx: &Context) -> i64 { - req_attr( - self.get_attr_arith_imm_imm(ctx), - "volt.arith_imm", - "arith_imm_imm", - ) - .value() - } -} - -/// Integer comparison: `volt.icmp lt %lhs, %rhs : volt.bool`. -#[pliron_op( - name = "volt.icmp", - format = "attr($icmp_kind, $CmpOpAttr) ` ` $0 `, ` $1 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (icmp_kind: CmpOpAttr), - verifier = "succ", -)] -pub struct IcmpOp; - -impl IcmpOp { - /// Build a comparison. - pub fn new(ctx: &mut Context, kind: CmpOp, lhs: Value, rhs: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![lhs, rhs], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_icmp_kind(ctx, CmpOpAttr::new(kind)); - op - } - - /// The comparison operator. - pub fn kind(&self, ctx: &Context) -> CmpOp { - req_attr(self.get_attr_icmp_kind(ctx), "volt.icmp", "icmp_kind").value() - } - - /// Left operand. - pub fn lhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Right operand. - pub fn rhs(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(1) - } -} - -/// Conditional select: `volt.select %c, %then, %else : T`. -#[pliron_op( - name = "volt.select", - format = "$0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct SelectOp; - -impl SelectOp { - /// Build a select. - pub fn new(ctx: &mut Context, cond: Value, then: Value, else_: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![cond, then, else_], - Vec::new(), - 0, - ); - Self { op } - } - - /// Condition, then-value, else-value. - pub fn operands3(&self, ctx: &Context) -> (Value, Value, Value) { - let op = self.get_operation().deref(ctx); - (op.get_operand(0), op.get_operand(1), op.get_operand(2)) - } -} - -/// Struct construction from field values. -#[pliron_op( - name = "volt.struct_new", - format = "operands(CharSpace(`,`)) ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct StructNewOp; - -impl StructNewOp { - /// Build a struct value. - pub fn new(ctx: &mut Context, fields: Vec, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - fields, - Vec::new(), - 0, - ); - Self { op } - } - - /// Field values. - pub fn fields(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().collect() - } -} - -/// Struct field extraction: `volt.extract %agg 1 : T`. -#[pliron_op( - name = "volt.extract", - format = "$0 attr($extract_index, $ImmAttr) ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (extract_index: ImmAttr), - verifier = "succ", -)] -pub struct ExtractOp; - -impl ExtractOp { - /// Build a field extraction. - pub fn new(ctx: &mut Context, aggregate: Value, index: u32, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![aggregate], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_extract_index(ctx, ImmAttr::new(index as i64)); - op - } - - /// The aggregate value. - pub fn aggregate(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Field index. - pub fn index(&self, ctx: &Context) -> u32 { - req_attr( - self.get_attr_extract_index(ctx), - "volt.extract", - "extract_index", - ) - .value() as u32 - } -} - -/// Type conversion (int/float resize, int<->float, bitcast flavors). -#[pliron_op( - name = "volt.convert", - format = "$0 ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct ConvertOp; - -impl ConvertOp { - /// Build a conversion. - pub fn new(ctx: &mut Context, value: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value], - Vec::new(), - 0, - ); - Self { op } - } - - /// The converted value. - pub fn value(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Low-precision float decode: `volt.decode_low_float bf16 %v : T`. -#[pliron_op( - name = "volt.decode_low_float", - format = "attr($decode_format, $LowFloatAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (decode_format: LowFloatAttr), - verifier = "succ", -)] -pub struct DecodeLowFloatOp; - -impl DecodeLowFloatOp { - /// Build a decode. - pub fn new(ctx: &mut Context, format: LowFloatFormat, value: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_decode_format(ctx, LowFloatAttr::new(format)); - op - } - - /// The source format. - pub fn float_format(&self, ctx: &Context) -> LowFloatFormat { - req_attr( - self.get_attr_decode_format(ctx), - "volt.decode_low_float", - "decode_format", - ) - .value() - } - - /// The encoded value. - pub fn value(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Low-precision float encode. -#[pliron_op( - name = "volt.encode_low_float", - format = "attr($encode_format, $LowFloatAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (encode_format: LowFloatAttr), - verifier = "succ", -)] -pub struct EncodeLowFloatOp; - -impl EncodeLowFloatOp { - /// Build an encode. - pub fn new(ctx: &mut Context, format: LowFloatFormat, value: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_encode_format(ctx, LowFloatAttr::new(format)); - op - } - - /// The target format. - pub fn float_format(&self, ctx: &Context) -> LowFloatFormat { - req_attr( - self.get_attr_encode_format(ctx), - "volt.encode_low_float", - "encode_format", - ) - .value() - } - - /// The decoded value. - pub fn value(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// NVFP4 dequantize: value + block/global scales. -#[pliron_op( - name = "volt.dequantize_nvfp4", - format = "attr($dequant_format, $LowFloatAttr) ` ` attr($dequant_block_app, $ScaleAppAttr) ` ` attr($dequant_global_app, $ScaleAppAttr) ` ` $0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (dequant_format: LowFloatAttr, dequant_block_app: ScaleAppAttr, dequant_global_app: ScaleAppAttr), - verifier = "succ", -)] -pub struct DequantizeNvfp4Op; - -impl DequantizeNvfp4Op { - /// Build a dequantize. - #[allow(clippy::too_many_arguments)] - pub fn new( - ctx: &mut Context, - format: LowFloatFormat, - value: Value, - block_scale: Value, - global_scale: Value, - block_app: ScaleApplication, - global_app: ScaleApplication, - ty: TypeHandle, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value, block_scale, global_scale], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_dequant_format(ctx, LowFloatAttr::new(format)); - op.set_attr_dequant_block_app(ctx, ScaleAppAttr::new(block_app)); - op.set_attr_dequant_global_app(ctx, ScaleAppAttr::new(global_app)); - op - } - - /// Conversion parameters and operands (value, block scale, global scale). - pub fn parts( - &self, - ctx: &Context, - ) -> ( - LowFloatFormat, - ScaleApplication, - ScaleApplication, - Value, - Value, - Value, - ) { - let op = self.get_operation(); - let format = req_attr( - self.get_attr_dequant_format(ctx), - "volt.dequantize_nvfp4", - "dequant_format", - ) - .value(); - let block_app = req_attr( - self.get_attr_dequant_block_app(ctx), - "volt.dequantize_nvfp4", - "dequant_block_app", - ) - .value(); - let global_app = req_attr( - self.get_attr_dequant_global_app(ctx), - "volt.dequantize_nvfp4", - "dequant_global_app", - ) - .value(); - let op = op.deref(ctx); - ( - format, - block_app, - global_app, - op.get_operand(0), - op.get_operand(1), - op.get_operand(2), - ) - } -} - -/// NVFP4 quantize: value + block/global scales. -#[pliron_op( - name = "volt.quantize_nvfp4", - format = "attr($quant_format, $LowFloatAttr) ` ` attr($quant_block_app, $ScaleAppAttr) ` ` attr($quant_global_app, $ScaleAppAttr) ` ` $0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (quant_format: LowFloatAttr, quant_block_app: ScaleAppAttr, quant_global_app: ScaleAppAttr), - verifier = "succ", -)] -pub struct QuantizeNvfp4Op; - -impl QuantizeNvfp4Op { - /// Build a quantize. - #[allow(clippy::too_many_arguments)] - pub fn new( - ctx: &mut Context, - format: LowFloatFormat, - value: Value, - block_scale: Value, - global_scale: Value, - block_app: ScaleApplication, - global_app: ScaleApplication, - ty: TypeHandle, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value, block_scale, global_scale], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_quant_format(ctx, LowFloatAttr::new(format)); - op.set_attr_quant_block_app(ctx, ScaleAppAttr::new(block_app)); - op.set_attr_quant_global_app(ctx, ScaleAppAttr::new(global_app)); - op - } - - /// Conversion parameters and operands (value, block scale, global scale). - pub fn parts( - &self, - ctx: &Context, - ) -> ( - LowFloatFormat, - ScaleApplication, - ScaleApplication, - Value, - Value, - Value, - ) { - let op = self.get_operation(); - let format = req_attr( - self.get_attr_quant_format(ctx), - "volt.quantize_nvfp4", - "quant_format", - ) - .value(); - let block_app = req_attr( - self.get_attr_quant_block_app(ctx), - "volt.quantize_nvfp4", - "quant_block_app", - ) - .value(); - let global_app = req_attr( - self.get_attr_quant_global_app(ctx), - "volt.quantize_nvfp4", - "quant_global_app", - ) - .value(); - let op = op.deref(ctx); - ( - format, - block_app, - global_app, - op.get_operand(0), - op.get_operand(1), - op.get_operand(2), - ) - } -} - -/// Unary float/int op: `volt.unary sqrt %v : T`. -#[pliron_op( - name = "volt.unary", - format = "attr($unary_kind, $UnaryOpAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (unary_kind: UnaryOpAttr), - verifier = "succ", -)] -pub struct UnaryOp; - -impl UnaryOp { - /// Build a unary op. - pub fn new( - ctx: &mut Context, - kind: super::super::function::UnaryOp, - value: Value, - ty: TypeHandle, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![value], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_unary_kind(ctx, UnaryOpAttr::new(kind)); - op - } - - /// The unary operator. - pub fn kind(&self, ctx: &Context) -> super::super::function::UnaryOp { - req_attr(self.get_attr_unary_kind(ctx), "volt.unary", "unary_kind").value() - } - - /// The operand. - pub fn value(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Stack allocation of one element. -#[pliron_op( - name = "volt.alloca", - format = "attr($alloca_elem, $TypeAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (alloca_elem: TypeAttr), - verifier = "succ", -)] -pub struct AllocaOp; - -impl AllocaOp { - /// Build an alloca. - pub fn new(ctx: &mut Context, elem: TypeHandle, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_alloca_elem(ctx, TypeAttr::new(elem)); - op - } - - /// Element type. - pub fn elem_type(&self, ctx: &Context) -> TypeHandle { - TypeHandle::from(req_attr( - self.get_attr_alloca_elem(ctx), - "volt.alloca", - "alloca_elem", - )) - } -} - -/// Direct call: `volt.call @"sym"(%args) : T`. -/// Operand 0.. are the call arguments; there are 0 or 1 results. -#[pliron_op( - name = "volt.call", - format = "`@` attr($call_callee, $StringAttr) `(` operands(CharSpace(`,`)) `)` ` ` attr($call_info, $CallInfoAttr) ` : ` types(CharSpace(`,`))", - attributes = (call_callee: StringAttr, call_info: CallInfoAttr), - verifier = "succ", -)] -pub struct CallOp; - -impl CallOp { - /// Build a direct call. - pub fn new( - ctx: &mut Context, - callee: &str, - args: Vec, - results: Vec, - info: CallInfo, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - results, - args, - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_call_callee(ctx, StringAttr::new(String::from(callee))); - op.set_attr_call_info(ctx, CallInfoAttr::new(info)); - op - } - - /// Callee symbol name. - pub fn callee(&self, ctx: &Context) -> String { - req_attr(self.get_attr_call_callee(ctx), "volt.call", "call_callee") - .as_str() - .into() - } - - /// Call ABI details. - pub fn info(&self, ctx: &Context) -> CallInfo { - req_attr(self.get_attr_call_info(ctx), "volt.call", "call_info") - .info() - .clone() - } - - /// Call arguments. - pub fn args(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().collect() - } -} - -/// Indirect call through a function pointer. Operand 0 is the target. -#[pliron_op( - name = "volt.call_indirect", - format = "operands(CharSpace(`,`)) ` ` attr($call_indirect_info, $CallInfoAttr) ` : ` types(CharSpace(`,`))", - attributes = (call_indirect_info: CallInfoAttr), - verifier = "succ", -)] -pub struct CallIndirectOp; - -impl CallIndirectOp { - /// Build an indirect call. - pub fn new( - ctx: &mut Context, - target: Value, - args: Vec, - results: Vec, - info: CallInfo, - ) -> Self { - let mut operands = alloc::vec![target]; - operands.extend(args); - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - results, - operands, - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_call_indirect_info(ctx, CallInfoAttr::new(info)); - op - } - - /// Call target. - pub fn target(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Call ABI details. - pub fn info(&self, ctx: &Context) -> CallInfo { - req_attr( - self.get_attr_call_indirect_info(ctx), - "volt.call_indirect", - "call_indirect_info", - ) - .info() - .clone() - } - - /// Call arguments (excluding the target). - pub fn args(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().skip(1).collect() - } -} - -/// Address of a global symbol. -#[pliron_op( - name = "volt.global_addr", - format = "`@` attr($global_symbol, $StringAttr) ` ` attr($global_via_got, $BoolAttr) ` : ` type($0)", - interfaces = [NOpdsInterface<0>, OneResultInterface], - attributes = (global_symbol: StringAttr, global_via_got: BoolAttr), - verifier = "succ", -)] -pub struct GlobalAddrOp; - -impl GlobalAddrOp { - /// Build a global address. - pub fn new(ctx: &mut Context, symbol: &str, via_got: bool, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_global_symbol(ctx, StringAttr::new(String::from(symbol))); - op.set_attr_global_via_got(ctx, BoolAttr::new(via_got)); - op - } - - /// Symbol name. - pub fn symbol(&self, ctx: &Context) -> String { - req_attr( - self.get_attr_global_symbol(ctx), - "volt.global_addr", - "global_symbol", - ) - .as_str() - .into() - } - - /// Route through the GOT. - pub fn via_got(&self, ctx: &Context) -> bool { - req_attr( - self.get_attr_global_via_got(ctx), - "volt.global_addr", - "global_via_got", - ) - .into() - } -} - -/// Memory load: `volt.load false %ptr : T`. -#[pliron_op( - name = "volt.load", - format = "attr($load_volatile, $BoolAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (load_volatile: BoolAttr), - verifier = "succ", -)] -pub struct LoadOp; - -impl LoadOp { - /// Build a load. - pub fn new(ctx: &mut Context, ptr: Value, volatile: bool, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![ptr], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_load_volatile(ctx, BoolAttr::new(volatile)); - op - } - - /// Pointer operand. - pub fn ptr(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Volatile access. - pub fn volatile(&self, ctx: &Context) -> bool { - req_attr( - self.get_attr_load_volatile(ctx), - "volt.load", - "load_volatile", - ) - .into() - } -} - -/// Memory store: `volt.store false %value, %ptr`. -#[pliron_op( - name = "volt.store", - format = "attr($store_volatile, $BoolAttr) ` ` $0 `, ` $1", - interfaces = [NOpdsInterface<2>, NResultsInterface<0>], - attributes = (store_volatile: BoolAttr), - verifier = "succ", -)] -pub struct StoreOp; - -impl StoreOp { - /// Build a store. - pub fn new(ctx: &mut Context, value: Value, ptr: Value, volatile: bool) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - alloc::vec![value, ptr], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_store_volatile(ctx, BoolAttr::new(volatile)); - op - } - - /// Stored value and pointer. - pub fn parts(&self, ctx: &Context) -> (Value, Value) { - let op = self.get_operation().deref(ctx); - (op.get_operand(0), op.get_operand(1)) - } - - /// Volatile access. - pub fn volatile(&self, ctx: &Context) -> bool { - req_attr( - self.get_attr_store_volatile(ctx), - "volt.store", - "store_volatile", - ) - .into() - } -} - -/// Cache prefetch hint. -#[pliron_op( - name = "volt.prefetch", - format = "$0", - interfaces = [NOpdsInterface<1>, NResultsInterface<0>], - verifier = "succ", -)] -pub struct PrefetchOp; - -impl PrefetchOp { - /// Build a prefetch. - pub fn new(ctx: &mut Context, ptr: Value) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - alloc::vec![ptr], - Vec::new(), - 0, - ); - Self { op } - } - - /// Pointer operand. - pub fn ptr(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Start of variadic argument access. -#[pliron_op( - name = "volt.va_start", - format = "$0", - interfaces = [NOpdsInterface<1>, NResultsInterface<0>], - verifier = "succ", -)] -pub struct VaStartOp; - -impl VaStartOp { - /// Build a va_start. - pub fn new(ctx: &mut Context, list: Value) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - alloc::vec![list], - Vec::new(), - 0, - ); - Self { op } - } - - /// Argument list pointer. - pub fn list(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Fetch the next variadic argument. -#[pliron_op( - name = "volt.va_arg", - format = "attr($va_arg_ty, $TypeAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (va_arg_ty: TypeAttr), - verifier = "succ", -)] -pub struct VaArgOp; - -impl VaArgOp { - /// Build a va_arg. - pub fn new(ctx: &mut Context, list: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![list], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_va_arg_ty(ctx, TypeAttr::new(ty)); - op - } - - /// Argument list pointer. - pub fn list(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// End of variadic argument access. -#[pliron_op( - name = "volt.va_end", - format = "$0", - interfaces = [NOpdsInterface<1>, NResultsInterface<0>], - verifier = "succ", -)] -pub struct VaEndOp; - -impl VaEndOp { - /// Build a va_end. - pub fn new(ctx: &mut Context, list: Value) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - alloc::vec![list], - Vec::new(), - 0, - ); - Self { op } - } - - /// Argument list pointer. - pub fn list(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Fused dot product: `acc + dot(a, b)`. -#[pliron_op( - name = "volt.dot", - format = "$0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct DotOp; - -impl DotOp { - /// Build a dot product. - pub fn new(ctx: &mut Context, acc: Value, a: Value, b: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![acc, a, b], - Vec::new(), - 0, - ); - Self { op } - } - - /// Accumulator and linear-algebra operands. - pub fn operands3(&self, ctx: &Context) -> (Value, Value, Value) { - let op = self.get_operation().deref(ctx); - (op.get_operand(0), op.get_operand(1), op.get_operand(2)) - } -} - -/// Vector reduction with a binary operator. -#[pliron_op( - name = "volt.reduce", - format = "attr($reduce_kind, $BinOpAttr) ` ` $0 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (reduce_kind: BinOpAttr), - verifier = "succ", -)] -pub struct ReduceOp; - -impl ReduceOp { - /// Build a reduction. - pub fn new(ctx: &mut Context, kind: BinOp, vector: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![vector], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_reduce_kind(ctx, BinOpAttr::new(kind)); - op - } - - /// The reduction operator. - pub fn kind(&self, ctx: &Context) -> BinOp { - req_attr(self.get_attr_reduce_kind(ctx), "volt.reduce", "reduce_kind").value() - } - - /// The reduced vector. - pub fn vector(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Broadcast a scalar to a vector. -#[pliron_op( - name = "volt.splat", - format = "$0 ` : ` type($0)", - interfaces = [OneResultInterface], - verifier = "succ", -)] -pub struct SplatOp; - -impl SplatOp { - /// Build a splat. - pub fn new(ctx: &mut Context, scalar: Value, ty: TypeHandle) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![scalar], - Vec::new(), - 0, - ); - Self { op } - } - - /// The broadcast scalar. - pub fn scalar(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } -} - -/// Matrix multiply-accumulate over three buffers. -#[pliron_op( - name = "volt.matmul", - format = "attr($matmul_config, $MatMulAttr) ` ` $0 `, ` $1 `, ` $2 ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (matmul_config: MatMulAttr), - verifier = "succ", -)] -pub struct MatMulOp; - -impl MatMulOp { - /// Build a matmul. - pub fn new( - ctx: &mut Context, - a: Value, - b: Value, - c: Value, - config: MatMulDesc, - ty: TypeHandle, - ) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - alloc::vec![a, b, c], - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_matmul_config(ctx, MatMulAttr::new(config)); - op - } - - /// Configuration. - pub fn config(&self, ctx: &Context) -> MatMulDesc { - req_attr( - self.get_attr_matmul_config(ctx), - "volt.matmul", - "matmul_config", - ) - .desc() - .clone() - } - - /// Buffer operands (a, b, c). - pub fn operands3(&self, ctx: &Context) -> (Value, Value, Value) { - let op = self.get_operation().deref(ctx); - (op.get_operand(0), op.get_operand(1), op.get_operand(2)) - } -} - -/// Execution barrier. -#[pliron_op( - name = "volt.barrier", - format = "attr($barrier_scope, $BarrierAttr)", - interfaces = [NOpdsInterface<0>, NResultsInterface<0>], - attributes = (barrier_scope: BarrierAttr), - verifier = "succ", -)] -pub struct BarrierOp; - -impl BarrierOp { - /// Build a barrier. - pub fn new(ctx: &mut Context, scope: BarrierScope) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - Vec::new(), - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_barrier_scope(ctx, BarrierAttr::new(scope)); - op - } - - /// Barrier scope. - pub fn scope(&self, ctx: &Context) -> BarrierScope { - req_attr( - self.get_attr_barrier_scope(ctx), - "volt.barrier", - "barrier_scope", - ) - .value() - } -} - -/// Atomic read-modify-write. Operands are `ptr, value`, plus `compare` -/// for compare-exchange. -#[pliron_op( - name = "volt.atomic_rmw", - format = "attr($atomic_kind, $AtomicOpAttr) ` ` attr($atomic_ordering, $AtomicOrderingAttr) ` ` attr($atomic_scope, $AtomicScopeAttr) ` ` operands(CharSpace(`,`)) ` : ` type($0)", - interfaces = [OneResultInterface], - attributes = (atomic_kind: AtomicOpAttr, atomic_ordering: AtomicOrderingAttr, atomic_scope: AtomicScopeAttr), - verifier = "succ", -)] -pub struct AtomicRmwOp; - -impl AtomicRmwOp { - /// Build an atomic RMW. - pub fn new( - ctx: &mut Context, - kind: AtomicOp, - ptr: Value, - value: Value, - compare: Option, - ordering: AtomicOrdering, - scope: AtomicScope, - ty: TypeHandle, - ) -> Self { - let mut operands = alloc::vec![ptr, value]; - if let Some(c) = compare { - operands.push(c); - } - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - alloc::vec![ty], - operands, - Vec::new(), - 0, - ); - let op = Self { op }; - op.set_attr_atomic_kind(ctx, AtomicOpAttr::new(kind)); - op.set_attr_atomic_ordering(ctx, AtomicOrderingAttr::new(ordering)); - op.set_attr_atomic_scope(ctx, AtomicScopeAttr::new(scope)); - op - } - - /// The atomic operator. - pub fn kind(&self, ctx: &Context) -> AtomicOp { - req_attr( - self.get_attr_atomic_kind(ctx), - "volt.atomic_rmw", - "atomic_kind", - ) - .value() - } - - /// Ordering. - pub fn ordering(&self, ctx: &Context) -> AtomicOrdering { - req_attr( - self.get_attr_atomic_ordering(ctx), - "volt.atomic_rmw", - "atomic_ordering", - ) - .value() - } - - /// Scope. - pub fn scope(&self, ctx: &Context) -> AtomicScope { - req_attr( - self.get_attr_atomic_scope(ctx), - "volt.atomic_rmw", - "atomic_scope", - ) - .value() - } - - /// Pointer, value, and optional compare operands. - pub fn operands_rmw(&self, ctx: &Context) -> (Value, Value, Option) { - let op = self.get_operation().deref(ctx); - let n = op.get_num_operands(); - ( - op.get_operand(0), - op.get_operand(1), - if n > 2 { Some(op.get_operand(2)) } else { None }, - ) - } -} - -/// Unconditional branch: `volt.jump ^target(%args)`. -#[pliron_op( - name = "volt.jump", - format = "succ($0) `(` operands(CharSpace(`,`)) `)`", - interfaces = [IsTerminatorInterface, NResultsInterface<0>], - verifier = "succ", -)] -pub struct JumpOp; - -impl JumpOp { - /// Build a jump. - pub fn new(ctx: &mut Context, target: Ptr, args: Vec) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - args, - alloc::vec![target], - 0, - ); - Self { op } - } - - /// Jump target. - pub fn target(&self, ctx: &Context) -> Ptr { - self.get_operation().deref(ctx).get_successor(0) - } - - /// Block arguments. - pub fn args(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().collect() - } -} - -/// Conditional branch with per-successor operands. -/// -/// Stored as one successor list `[then, else]`, one operand list -/// `[cond, ...then_args, ...else_args]`, and a `then_args` count -/// attribute splitting the two argument groups. -/// -/// Syntax: `volt.cond_br %cond, ^then(%a), ^else(%b)`. -#[pliron_op( - name = "volt.cond_br", - interfaces = [IsTerminatorInterface, NResultsInterface<0>], - attributes = (cond_then_args: ImmAttr), - verifier = "succ", -)] -pub struct CondBrOp; - -impl CondBrOp { - /// Build a conditional branch. - pub fn new( - ctx: &mut Context, - cond: Value, - then: Ptr, - then_args: Vec, - else_: Ptr, - else_args: Vec, - ) -> Self { - let n_then = then_args.len() as i64; - let mut operands = alloc::vec![cond]; - operands.extend(then_args); - operands.extend(else_args); - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - operands, - alloc::vec![then, else_], - 0, - ); - let op = Self { op }; - op.set_attr_cond_then_args(ctx, ImmAttr::new(n_then)); - op - } - - /// Branch condition. - pub fn cond(&self, ctx: &Context) -> Value { - self.get_operation().deref(ctx).get_operand(0) - } - - /// Number of then-successor arguments. - fn n_then(&self, ctx: &Context) -> usize { - req_attr( - self.get_attr_cond_then_args(ctx), - "volt.cond_br", - "cond_then_args", - ) - .value() as usize - } - - /// Then destination. - pub fn then_dest(&self, ctx: &Context) -> Ptr { - self.get_operation().deref(ctx).get_successor(0) - } - - /// Else destination. - pub fn else_dest(&self, ctx: &Context) -> Ptr { - self.get_operation().deref(ctx).get_successor(1) - } - - /// Then arguments. - pub fn then_args(&self, ctx: &Context) -> Vec { - let n = self.n_then(ctx); - self.get_operation() - .deref(ctx) - .operands() - .skip(1) - .take(n) - .collect() - } - - /// Else arguments. - pub fn else_args(&self, ctx: &Context) -> Vec { - let n = self.n_then(ctx); - self.get_operation() - .deref(ctx) - .operands() - .skip(1 + n) - .collect() - } -} - -impl Printable for CondBrOp { - fn fmt( - &self, - ctx: &Context, - state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - write!( - f, - "{} {}, ^{}({}), ^{}({})", - self.get_opid().print(ctx, state), - self.cond(ctx).print(ctx, state), - self.then_dest(ctx).unique_name(ctx), - comma_values(ctx, state, &self.then_args(ctx)), - self.else_dest(ctx).unique_name(ctx), - comma_values(ctx, state, &self.else_args(ctx)), - ) - } -} - -fn comma_values(ctx: &Context, state: &printable::State, values: &[Value]) -> String { - let mut out = String::new(); - for (i, v) in values.iter().enumerate() { - if i != 0 { - out.push_str(", "); - } - out.push_str(&alloc::format!("{}", v.print(ctx, state))); - } - out -} - -impl Parsable for CondBrOp { - type Arg = Vec<(Identifier, Location)>; - type Parsed = OpObj; - - fn parse<'a>( - state_stream: &mut StateStream<'a>, - results: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - let loc = state_stream.loc(); - if !results.is_empty() { - input_err!(loc, "volt.cond_br cannot define SSA values")?; - } - let cond = ssa_opd_parser().parse_stream(state_stream).into_result()?.0; - spaced(token(',')) - .parse_stream(state_stream) - .into_result()?; - let then = block_opd_parser() - .parse_stream(state_stream) - .into_result()? - .0; - let then_args = delimited_list_parser('(', ')', ',', ssa_opd_parser()) - .parse_stream(state_stream) - .into_result()? - .0; - spaced(token(',')) - .parse_stream(state_stream) - .into_result()?; - let else_ = block_opd_parser() - .parse_stream(state_stream) - .into_result()? - .0; - let else_args = delimited_list_parser('(', ')', ',', ssa_opd_parser()) - .parse_stream(state_stream) - .into_result()? - .0; - let op = Self::new( - state_stream.state.ctx, - cond, - then, - then_args, - else_, - else_args, - ); - process_parsed_ssa_defs(state_stream, &results, op.get_operation())?; - Ok(OpObj::new(op)).into_parse_result() - } -} - -/// Function return with up to N values. -#[pliron_op( - name = "volt.ret", - format = "operands(CharSpace(`,`))", - interfaces = [IsTerminatorInterface, NResultsInterface<0>], - verifier = "succ", -)] -pub struct RetOp; - -impl RetOp { - /// Build a return. - pub fn new(ctx: &mut Context, values: Vec) -> Self { - let op = Operation::new( - ctx, - Self::get_concrete_op_info(), - Vec::new(), - values, - Vec::new(), - 0, - ); - Self { op } - } - - /// Returned values. - pub fn values(&self, ctx: &Context) -> Vec { - self.get_operation().deref(ctx).operands().collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::dialect::types::{BoolType, IntType}; - use pliron::{ - builtin::{op_interfaces::OneRegionInterface, ops::FuncOp}, - parsable::parse_from_str, - result::ExpectOk, - }; - - #[test] - fn arith_print_parse() { - let ctx = &mut Context::new(); - let i32t = IntType::get_type(ctx, true, 32); - let boolt = BoolType::get_type(ctx); - let func_ty = pliron::builtin::types::FunctionType::get(ctx, vec![i32t, i32t], vec![boolt]); - let func = FuncOp::new(ctx, pliron::ident!("f"), func_ty); - let entry = func.get_entry_block(ctx); - let a = entry.deref(ctx).get_argument(0); - let b = entry.deref(ctx).get_argument(1); - let add = ArithOp::new(ctx, BinOp::Add, a, b, i32t); - add.get_operation().insert_at_back(entry, ctx); - let text = alloc::format!( - "{}", - add.get_operation() - .deref(ctx) - .print(ctx, &Default::default()) - ); - assert!(text.contains("volt.arith"), "{text}"); - assert!(text.contains("add"), "{text}"); - let _ = boolt; - } - - #[test] - fn cond_br_round_trip() { - let ctx = &mut Context::new(); - let boolt = BoolType::get_type(ctx); - let func_ty = pliron::builtin::types::FunctionType::get(ctx, vec![boolt], vec![]); - let func = FuncOp::new(ctx, pliron::ident!("f"), func_ty); - let entry = func.get_entry_block(ctx); - let region = func.get_region(ctx); - let then_bb = BasicBlock::new(ctx, Some(pliron::ident!("then")), vec![]); - then_bb.insert_at_back(region, ctx); - let else_bb = BasicBlock::new(ctx, Some(pliron::ident!("else")), vec![]); - else_bb.insert_at_back(region, ctx); - let cond = entry.deref(ctx).get_argument(0); - let br = CondBrOp::new(ctx, cond, then_bb, vec![], else_bb, vec![]); - br.get_operation().insert_at_back(entry, ctx); - let func_text = alloc::format!( - "{}", - func.get_operation() - .deref(ctx) - .print(ctx, &Default::default()) - ); - let parsed = parse_from_str(Operation::top_level_parser(), ctx, &func_text).expect_ok(ctx); - let _ = parsed; - } -} diff --git a/src/ir/src/dialect/types.rs b/src/ir/src/dialect/types.rs deleted file mode 100644 index cae2eeb..0000000 --- a/src/ir/src/dialect/types.rs +++ /dev/null @@ -1,450 +0,0 @@ -//! The volt type system as pliron types. -//! -//! Every variant of the legacy [`TypeKind`](crate::types::TypeKind) has a -//! corresponding pliron type in the `volt` dialect: -//! -//! | volt type | example syntax | -//! |------------------------|-----------------------------------------| -//! | [`BoolType`] | `volt.bool` | -//! | [`IntType`] | `volt.int` | -//! | [`FloatType`] | `volt.float` | -//! | [`PtrType`] | `volt.ptr(global)` | -//! | [`VectorType`] | `volt.vector<4 x volt.int>` | -//! | [`ArrayType`] | `volt.array[8 x volt.float]` | -//! | [`SliceType`] | `volt.slice[]volt.int` | -//! | [`StructType`] | `volt.struct{...}` | -//! -//! [`describe`] inspects any [`TypeHandle`] and returns an owned -//! [`VoltKind`], which is the main entry point for instruction selection. - -use alloc::vec::Vec; -use core::hash::Hash; - -use pliron::{ - combine::{ - Parser, attempt, choice, - parser::char::{spaces, string}, - parser::token::token, - }, - context::Context, - derive::pliron_type, - parsable::{Parsable, ParseResult, StateStream}, - printable::{self, Printable}, - r#type::TypeHandle, -}; - -use super::super::types::{AddressSpace, FloatKind}; - -/// A boolean value. -#[pliron_type(name = "volt.bool", generate_get = true, format, verifier = "succ")] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy, Default)] -pub struct BoolType; - -impl BoolType { - /// Intern (or fetch) the boolean type. - pub fn get_type(ctx: &Context) -> TypeHandle { - Self::get(ctx).into() - } -} - -/// A fixed-width integer. `signed` selects the `i`/`u` flavor. -#[pliron_type( - name = "volt.int", - generate_get = true, - format = "`<` $signed `, ` $bits `>`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct IntType { - signed: bool, - bits: u32, -} - -impl IntType { - /// Intern (or fetch) an integer type. - pub fn get_type(ctx: &Context, signed: bool, bits: u32) -> TypeHandle { - Self::get(ctx, signed, bits).into() - } - - /// Is this the signed flavor? - pub fn is_signed(&self) -> bool { - self.signed - } - - /// Bit width. - pub fn bits(&self) -> u32 { - self.bits - } -} - -/// A floating-point scalar. Prints with the legacy keyword (`f32`, ...). -#[pliron_type( - name = "volt.float", - generate_get = true, - format = "`<` $kind `>`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct FloatType { - kind: FloatKind, -} - -impl FloatType { - /// Intern (or fetch) a float type. - pub fn get_type(ctx: &Context, kind: FloatKind) -> TypeHandle { - Self::get(ctx, kind).into() - } - - /// The float flavor. - pub fn kind(&self) -> FloatKind { - self.kind - } -} - -/// A pointer into an address space. -#[pliron_type( - name = "volt.ptr", - generate_get = true, - format = "`(` $space `)`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct PtrType { - space: AddressSpace, -} - -impl PtrType { - /// Intern (or fetch) a pointer type. - pub fn get_type(ctx: &Context, space: AddressSpace) -> TypeHandle { - Self::get(ctx, space).into() - } - - /// The address space. - pub fn space(&self) -> AddressSpace { - self.space - } -} - -/// A fixed-length SIMD vector. -#[pliron_type( - name = "volt.vector", - generate_get = true, - format = "`<` $len ` x ` $elem `>`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct VectorType { - len: u32, - elem: TypeHandle, -} - -impl VectorType { - /// Intern (or fetch) a vector type. - pub fn get_type(ctx: &Context, len: u32, elem: TypeHandle) -> TypeHandle { - Self::get(ctx, len, elem).into() - } - - /// Lane count. - pub fn len(&self) -> u32 { - self.len - } - - /// Lane type. - pub fn elem(&self) -> TypeHandle { - self.elem - } -} - -/// A fixed-length array. -#[pliron_type( - name = "volt.array", - generate_get = true, - format = "`[` $len ` x ` $elem `]`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct ArrayType { - len: u64, - elem: TypeHandle, -} - -impl ArrayType { - /// Intern (or fetch) an array type. - pub fn get_type(ctx: &Context, len: u64, elem: TypeHandle) -> TypeHandle { - Self::get(ctx, len, elem).into() - } - - /// Element count. - pub fn len(&self) -> u64 { - self.len - } - - /// Element type. - pub fn elem(&self) -> TypeHandle { - self.elem - } -} - -/// An unsized slice (fat pointer payload). -#[pliron_type( - name = "volt.slice", - generate_get = true, - format = "`[` `]` $elem", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone, Copy)] -pub struct SliceType { - elem: TypeHandle, -} - -impl SliceType { - /// Intern (or fetch) a slice type. - pub fn get_type(ctx: &Context, elem: TypeHandle) -> TypeHandle { - Self::get(ctx, elem).into() - } - - /// Element type. - pub fn elem(&self) -> TypeHandle { - self.elem - } -} - -/// A heterogeneous struct. -#[pliron_type( - name = "volt.struct", - generate_get = true, - format = "`{` vec($fields, CharSpace(`,`)) `}`", - verifier = "succ" -)] -#[derive(Hash, PartialEq, Eq, Debug, Clone)] -pub struct StructType { - fields: Vec, -} - -impl StructType { - /// Intern (or fetch) a struct type. - pub fn get_type(ctx: &Context, fields: Vec) -> TypeHandle { - Self::get(ctx, fields).into() - } - - /// Field types. - pub fn fields(&self) -> &[TypeHandle] { - &self.fields - } -} - -/// An owned description of any volt type, mirroring the legacy `TypeKind`. -/// This is the main inspection entry point for instruction selection. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum VoltKind { - /// `volt.bool`. - Bool, - /// `volt.int`. - Int { signed: bool, bits: u32 }, - /// `volt.float`. - Float(FloatKind), - /// `volt.ptr`. - Ptr(AddressSpace), - /// `volt.vector`. - Vector { len: u32, elem: TypeHandle }, - /// `volt.array`. - Array { len: u64, elem: TypeHandle }, - /// `volt.slice`. - Slice { elem: TypeHandle }, - /// `volt.struct`. - Struct(Vec), -} - -/// Describe a type handle as an owned [`VoltKind`]. Returns `None` for -/// non-volt types (e.g. `builtin.function`). -pub fn describe(ctx: &Context, ty: TypeHandle) -> Option { - let obj = ty.deref(ctx); - if let Some(t) = obj.downcast_ref::() { - let _ = t; - return Some(VoltKind::Bool); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Int { - signed: t.is_signed(), - bits: t.bits(), - }); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Float(t.kind())); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Ptr(t.space())); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Vector { - len: t.len(), - elem: t.elem(), - }); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Array { - len: t.len(), - elem: t.elem(), - }); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Slice { elem: t.elem() }); - } - if let Some(t) = obj.downcast_ref::() { - return Some(VoltKind::Struct(t.fields().to_vec())); - } - None -} - -/// Intern an owned [`VoltKind`] description. -pub fn intern(ctx: &Context, kind: &VoltKind) -> TypeHandle { - match kind { - VoltKind::Bool => BoolType::get_type(ctx), - VoltKind::Int { signed, bits } => IntType::get_type(ctx, *signed, *bits), - VoltKind::Float(k) => FloatType::get_type(ctx, *k), - VoltKind::Ptr(s) => PtrType::get_type(ctx, *s), - VoltKind::Vector { len, elem } => VectorType::get_type(ctx, *len, *elem), - VoltKind::Array { len, elem } => ArrayType::get_type(ctx, *len, *elem), - VoltKind::Slice { elem } => SliceType::get_type(ctx, *elem), - VoltKind::Struct(f) => StructType::get_type(ctx, f.clone()), - } -} - -// The `$kind` / `$space` format fields above need print/parse support for -// the legacy descriptor enums. These print with the legacy keywords. - -impl Printable for FloatKind { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let s = match self { - FloatKind::F32 => "f32", - FloatKind::F64 => "f64", - FloatKind::F16 => "f16", - FloatKind::F128 => "f128", - }; - f.write_str(s) - } -} - -impl Printable for AddressSpace { - fn fmt( - &self, - _ctx: &Context, - _state: &printable::State, - f: &mut core::fmt::Formatter<'_>, - ) -> core::fmt::Result { - let s = match self { - AddressSpace::Global => "global", - AddressSpace::Shared => "shared", - AddressSpace::Private => "private", - AddressSpace::Constant => "constant", - }; - f.write_str(s) - } -} - -impl Parsable for FloatKind { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(token('f').with(choice(( - string("32").map(|_| FloatKind::F32), - string("64").map(|_| FloatKind::F64), - attempt(string("128")).map(|_| FloatKind::F128), - string("16").map(|_| FloatKind::F16), - )))) - .parse_stream(state_stream) - .into() - } -} - -impl Parsable for AddressSpace { - type Arg = (); - type Parsed = Self; - fn parse<'a>( - state_stream: &mut StateStream<'a>, - _arg: Self::Arg, - ) -> ParseResult<'a, Self::Parsed> { - spaces() - .with(choice(( - string("global").map(|_| AddressSpace::Global), - string("shared").map(|_| AddressSpace::Shared), - string("private").map(|_| AddressSpace::Private), - string("constant").map(|_| AddressSpace::Constant), - ))) - .parse_stream(state_stream) - .into() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use pliron::{ - parsable::{Parsable, parse_from_str}, - result::ExpectOk, - }; - - fn ctx() -> Context { - Context::new() - } - - #[test] - fn int_type_round_trip() { - let ctx = &mut ctx(); - let ty = IntType::get_type(ctx, true, 32); - assert_eq!( - describe(ctx, ty), - Some(VoltKind::Int { - signed: true, - bits: 32 - }) - ); - let parsed = - parse_from_str(TypeHandle::parser(()), ctx, "volt.int").expect_ok(ctx); - assert_eq!(parsed, ty); - } - - #[test] - fn composite_type_round_trip() { - let ctx = &mut ctx(); - let i32t = IntType::get_type(ctx, true, 32); - let v = VectorType::get_type(ctx, 4, i32t); - let printed = alloc::format!("{}", v.print(ctx, &Default::default())); - let back = parse_from_str(TypeHandle::parser(()), ctx, &printed).expect_ok(ctx); - assert_eq!(back, v); - - let s = StructType::get_type(ctx, alloc::vec![i32t, BoolType::get_type(ctx)]); - let printed = alloc::format!("{}", s.print(ctx, &Default::default())); - let back = parse_from_str(TypeHandle::parser(()), ctx, &printed).expect_ok(ctx); - assert_eq!(back, s); - } - - #[test] - fn float_ptr_kinds() { - let ctx = &mut ctx(); - for (text, kind) in [ - ("volt.float", VoltKind::Float(FloatKind::F32)), - ("volt.float", VoltKind::Float(FloatKind::F128)), - ("volt.ptr(shared)", VoltKind::Ptr(AddressSpace::Shared)), - ( - "volt.array[8 x volt.float]", - VoltKind::Array { - len: 8, - elem: FloatType::get_type(ctx, FloatKind::F64), - }, - ), - ] { - let parsed = parse_from_str(TypeHandle::parser(()), ctx, text).expect_ok(ctx); - assert_eq!(describe(ctx, parsed), Some(kind), "{text}"); - } - } -} diff --git a/src/ir/src/expand.rs b/src/ir/src/expand.rs index 1cf498b..b38a306 100644 --- a/src/ir/src/expand.rs +++ b/src/ir/src/expand.rs @@ -4,8 +4,8 @@ use alloc::vec::Vec; use super::function::AttrTarget; use super::function::{ - BinOp, Block, CmpOp, Function, Inst, LowFloatConvert, MatMul, MatMulType, NvFp4Convert, Opcode, - Value, + BinOp, Block, CmpOp, ConvertKind, Function, Inst, LowFloatConvert, MatMul, MatMulType, + MemFlags, NvFp4Convert, Opcode, Value, }; use super::low_float::Format as LowFloatFormat; use super::nvfp4::ScaleApplication; @@ -138,19 +138,22 @@ pub fn expand_vector_lanes_except( // --- NumericBuilder ---------------------------------------------------------- struct NumericBuilder { - u8_t: Type, - u32_t: Type, + i8_t: Type, + i32_t: Type, f32_t: Type, bool_t: Type, out: Vec, } /// The bounded operations a scalar expansion may emit. +/// +/// Every integer here is a plain 32-bit bit pattern read as UNSIGNED: widening zero-extends, +/// right shifts are logical and magnitude compares are `Ult`/`Ugt`/.... impl NumericBuilder { fn common(_func: &mut Function, ct: &CommonTypes) -> Self { Self { - u8_t: ct.u8_t, - u32_t: ct.u32_t, + i8_t: ct.i8_t, + i32_t: ct.i32_t, f32_t: ct.f32_t, bool_t: ct.bool_t, out: Vec::new(), @@ -164,13 +167,13 @@ impl NumericBuilder { } fn constant(&mut self, func: &mut Function, value: u32) -> Value { - self.emit(func, self.u32_t, Opcode::Iconst(value as i64)) + self.emit(func, self.i32_t, Opcode::Iconst(value as i64)) } fn binary(&mut self, func: &mut Function, op: BinOp, lhs: Value, rhs: Value) -> Value { self.emit( func, - self.u32_t, + self.i32_t, Opcode::Arith(super::function::Arith { op, lhs, rhs }), ) } @@ -197,7 +200,7 @@ impl NumericBuilder { fn select(&mut self, func: &mut Function, cond: Value, then_v: Value, else_v: Value) -> Value { self.emit( func, - self.u32_t, + self.i32_t, Opcode::Select(super::function::Select { cond, then: then_v, @@ -206,14 +209,31 @@ impl NumericBuilder { ) } - fn convert(&mut self, func: &mut Function, ty: Type, value: Value) -> Value { + fn convert(&mut self, func: &mut Function, ty: Type, kind: ConvertKind, value: Value) -> Value { self.emit( func, ty, - Opcode::Convert(super::function::Convert { value }), + Opcode::Convert(super::function::Convert { value, kind }), ) } + /// `value` as the i32 holding its bits zero-extended: a narrower int (or bool) is widened + /// with `Zext`, a wider one narrowed with `Trunc`, and an i32 is used as it is. + fn as_i32(&mut self, func: &mut Function, value: Value) -> Value { + let bits = match func.types.type_kind(func.value_type(value)) { + TypeKind::Int(d) => u32::from(d.bits), + TypeKind::Bool => 1, + _ => return value, + }; + match bits.cmp(&32) { + core::cmp::Ordering::Less => self.convert(func, self.i32_t, ConvertKind::Zext, value), + core::cmp::Ordering::Equal => value, + core::cmp::Ordering::Greater => { + self.convert(func, self.i32_t, ConvertKind::Trunc, value) + } + } + } + fn reinterpret(&mut self, func: &mut Function, ty: Type, value: Value) -> Value { self.emit( func, @@ -249,7 +269,7 @@ impl NumericBuilder { let discarded = self.binary(func, BinOp::BitAnd, value, mask); let shift_minus_one = self.binary(func, BinOp::Sub, shift, one); let halfway = self.binary(func, BinOp::Shl, one, shift_minus_one); - let greater = self.compare(func, CmpOp::Gt, discarded, halfway); + let greater = self.compare(func, CmpOp::Ugt, discarded, halfway); let equal = self.compare(func, CmpOp::Eq, discarded, halfway); let odd_bits = self.binary(func, BinOp::BitAnd, retained, one); let zero = self.constant(func, 0); @@ -265,30 +285,30 @@ impl NumericBuilder { /// Types used by the scalar builder passes, interned once per pass run. struct CommonTypes { - u8_t: Type, - u16_t: Type, - u32_t: Type, + i8_t: Type, + i16_t: Type, + i32_t: Type, f32_t: Type, bool_t: Type, } fn intern_common(func: &mut Function) -> CommonTypes { - let u8_t = func.types.intern(int_ty_kind(8, false)); - let u16_t = func.types.intern(int_ty_kind(16, false)); - let u32_t = func.types.intern(int_ty_kind(32, false)); + let i8_t = func.types.intern(int_ty_kind(8)); + let i16_t = func.types.intern(int_ty_kind(16)); + let i32_t = func.types.intern(int_ty_kind(32)); let f32_t = func.types.intern(TypeKind::Float(FloatKind::F32)); let bool_t = func.types.intern(TypeKind::Bool); CommonTypes { - u8_t, - u16_t, - u32_t, + i8_t, + i16_t, + i32_t, f32_t, bool_t, } } -fn int_ty_kind(bits: u16, signed: bool) -> TypeKind { - TypeKind::Int(IntDesc { signed, bits }) +fn int_ty_kind(bits: u16) -> TypeKind { + TypeKind::Int(IntDesc { bits }) } /// Replace all low-float storage conversions with target-independent integer /// operations. Returns whether anything changed. @@ -352,7 +372,7 @@ pub fn expand_low_float(func: &mut Function) -> bool { continue; }; let expanded = - encode_low_float(func, &mut builder, &conversion, inst, ct.u16_t, ct.u8_t); + encode_low_float(func, &mut builder, &conversion, inst, ct.i16_t, ct.i8_t); func.replace_all_uses(old_result, expanded); func.retarget_attrs(AttrTarget::Value(old_result), AttrTarget::Value(expanded)); out.append(&mut builder.out); @@ -366,7 +386,7 @@ pub fn expand_low_float(func: &mut Function) -> bool { } fn decode_low_float(func: &mut Function, b: &mut NumericBuilder, conv: &LowFloatConvert) -> Value { - let payload = b.convert(func, b.u32_t, conv.value); + let payload = b.as_i32(func, conv.value); let bits = match conv.format { LowFloatFormat::Bf16 => b.shifted(func, BinOp::Shl, payload, 16), LowFloatFormat::F8E4M3 => decode_e4m3(func, b, payload), @@ -466,10 +486,10 @@ fn encode_low_float( b: &mut NumericBuilder, conv: &LowFloatConvert, old_inst: Inst, - u16_t: Type, - u8_t: Type, + i16_t: Type, + i8_t: Type, ) -> Value { - let bits = b.reinterpret(func, b.u32_t, conv.value); + let bits = b.reinterpret(func, b.i32_t, conv.value); let defining = func.defining_inst(bits).unwrap(); func.retarget_attrs(AttrTarget::Inst(old_inst), AttrTarget::Inst(defining)); let payload = match conv.format { @@ -478,11 +498,11 @@ fn encode_low_float( LowFloatFormat::F8E5M2 => encode_fp8(func, b, bits, false), }; let narrow_ty = if conv.format == LowFloatFormat::Bf16 { - u16_t + i16_t } else { - u8_t + i8_t }; - b.convert(func, narrow_ty, payload) + b.convert(func, narrow_ty, ConvertKind::Trunc, payload) } fn encode_bf16(func: &mut Function, b: &mut NumericBuilder, bits: Value) -> Value { @@ -516,11 +536,11 @@ fn encode_fp8(func: &mut Function, b: &mut NumericBuilder, bits: Value, e4m3: bo let bias_v = b.constant(func, shift_bias); let original_shift = b.binary(func, BinOp::Sub, bias_v, source_exponent); let hoist17 = b.constant(func, 24); - let shift_too_large = b.compare(func, CmpOp::Gt, original_shift, hoist17); + let shift_too_large = b.compare(func, CmpOp::Ugt, original_shift, hoist17); let hoist18 = b.constant(func, 24); let bounded_shift = b.select(func, shift_too_large, hoist18, original_shift); let hoist1 = b.constant(func, threshold); - let subnormal_path = b.compare(func, CmpOp::Lt, source_exponent, hoist1); + let subnormal_path = b.compare(func, CmpOp::Ult, source_exponent, hoist1); let hoist19 = b.constant(func, 24); let safe_shift = b.select(func, subnormal_path, bounded_shift, hoist19); let subnormal_rounded = b.round_right(func, significand, safe_shift); @@ -549,7 +569,7 @@ fn encode_fp8(func: &mut Function, b: &mut NumericBuilder, bits: Value, e4m3: bo let finite_magnitude = b.select(func, subnormal_path, subnormal, normal); let max_finite: u32 = if e4m3 { 0x7e } else { 0x7b }; let hoist4 = b.constant(func, max_finite); - let too_large = b.compare(func, CmpOp::Gt, finite_magnitude, hoist4); + let too_large = b.compare(func, CmpOp::Ugt, finite_magnitude, hoist4); let hoist22 = b.constant(func, max_finite); let saturated = b.select(func, too_large, hoist22, finite_magnitude); let finite = b.binary(func, BinOp::BitOr, sign, saturated); @@ -686,7 +706,7 @@ fn dequantize_nvfp4( b: &mut NumericBuilder, conv: &NvFp4Convert, ) -> NvFp4Expansion { - let carrier = b.convert(func, b.u32_t, conv.value); + let carrier = b.as_i32(func, conv.value); let nibble = b.masked(func, carrier, 0x0f); let sign_part = b.masked(func, nibble, 0x08); let sign = b.shifted(func, BinOp::Shl, sign_part, 28); @@ -711,7 +731,7 @@ fn dequantize_nvfp4( let value_bits = b.binary(func, BinOp::BitOr, sign, magnitude); let value = b.reinterpret(func, b.f32_t, value_bits); - let block_payload = b.convert(func, b.u32_t, conv.block_scale); + let block_payload = b.as_i32(func, conv.block_scale); let block_bits = decode_e4m3(func, b, block_payload); let block_scale = b.reinterpret(func, b.f32_t, block_bits); let local = b.float_binary( @@ -730,7 +750,7 @@ fn dequantize_nvfp4( // This bit round trip blocks a backend from fusing the final scale with an // original consumer. - let scaled_bits = b.reinterpret(func, b.u32_t, scaled); + let scaled_bits = b.reinterpret(func, b.i32_t, scaled); let result = b.reinterpret(func, b.f32_t, scaled_bits); NvFp4Expansion { result, @@ -743,7 +763,7 @@ fn quantize_nvfp4( b: &mut NumericBuilder, conv: &NvFp4Convert, ) -> NvFp4Expansion { - let block_payload = b.convert(func, b.u32_t, conv.block_scale); + let block_payload = b.as_i32(func, conv.block_scale); let block_bits = decode_e4m3(func, b, block_payload); let block_scale = b.reinterpret(func, b.f32_t, block_bits); let global_unscaled = b.float_binary( @@ -758,7 +778,7 @@ fn quantize_nvfp4( global_unscaled, block_scale, ); - let bits = b.reinterpret(func, b.u32_t, local_unscaled); + let bits = b.reinterpret(func, b.i32_t, local_unscaled); let semantic_inst = func.defining_inst(bits).unwrap(); let shifted_bits = b.shifted(func, BinOp::Shr, bits, 28); @@ -766,13 +786,13 @@ fn quantize_nvfp4( let magnitude = b.masked(func, bits, 0x7fff_ffff); let mut payload = b.constant(func, 0x07); for (relation, threshold_bits, threshold_payload) in [ - (CmpOp::Le, 0x40a0_0000u32, 0x06u32), - (CmpOp::Lt, 0x4060_0000, 0x05), - (CmpOp::Le, 0x4020_0000, 0x04), - (CmpOp::Lt, 0x3fe0_0000, 0x03), - (CmpOp::Le, 0x3fa0_0000, 0x02), - (CmpOp::Lt, 0x3f40_0000, 0x01), - (CmpOp::Le, 0x3e80_0000, 0x00), + (CmpOp::Ule, 0x40a0_0000u32, 0x06u32), + (CmpOp::Ult, 0x4060_0000, 0x05), + (CmpOp::Ule, 0x4020_0000, 0x04), + (CmpOp::Ult, 0x3fe0_0000, 0x03), + (CmpOp::Ule, 0x3fa0_0000, 0x02), + (CmpOp::Ult, 0x3f40_0000, 0x01), + (CmpOp::Ule, 0x3e80_0000, 0x00), ] { let hoist29 = b.constant(func, threshold_bits); let cond = b.compare(func, relation, magnitude, hoist29); @@ -781,10 +801,10 @@ fn quantize_nvfp4( } let signed_payload = b.binary(func, BinOp::BitOr, sign, payload); let hoist5 = b.constant(func, 0x7f80_0000); - let is_nan = b.compare(func, CmpOp::Gt, magnitude, hoist5); + let is_nan = b.compare(func, CmpOp::Ugt, magnitude, hoist5); let hoist31 = b.constant(func, 0x07); let canonical = b.select(func, is_nan, hoist31, signed_payload); - let result = b.convert(func, b.u8_t, canonical); + let result = b.convert(func, b.i8_t, ConvertKind::Trunc, canonical); NvFp4Expansion { result, semantic_inst, @@ -813,7 +833,7 @@ fn normalize_f32(func: &mut Function, b: &mut NumericBuilder, magnitude: Value) let mut exponent = exponent; for shift in [16u32, 8, 4, 2, 1] { let threshold_v = b.constant(func, 0x0100_0000u32 >> shift); - let needs_shift = b.compare(func, CmpOp::Lt, significand, threshold_v); + let needs_shift = b.compare(func, CmpOp::Ult, significand, threshold_v); let shifted_sig = b.shifted(func, BinOp::Shl, significand, shift); significand = b.select(func, needs_shift, shifted_sig, significand); let hoist7 = b.constant(func, shift as i64 as u32); @@ -851,7 +871,7 @@ fn round_division_quotient( let discarded = b.binary(func, BinOp::BitAnd, quotient, mask); let shift_minus_one = b.binary(func, BinOp::Sub, shift, one); let halfway = b.binary(func, BinOp::Shl, one, shift_minus_one); - let discarded_gt = b.compare(func, CmpOp::Gt, discarded, halfway); + let discarded_gt = b.compare(func, CmpOp::Ugt, discarded, halfway); let above_half = bool_as_u32(func, b, discarded_gt); let discarded_eq = b.compare(func, CmpOp::Eq, discarded, halfway); let at_half = bool_as_u32(func, b, discarded_eq); @@ -876,8 +896,8 @@ fn expand_f32_division( numerator: Value, denominator: Value, ) -> Value { - let numerator_bits = b.reinterpret(func, b.u32_t, numerator); - let denominator_bits = b.reinterpret(func, b.u32_t, denominator); + let numerator_bits = b.reinterpret(func, b.i32_t, numerator); + let denominator_bits = b.reinterpret(func, b.i32_t, denominator); let numerator_magnitude = b.masked(func, numerator_bits, 0x7fff_ffff); let denominator_magnitude = b.masked(func, denominator_bits, 0x7fff_ffff); let signs = b.binary(func, BinOp::BitXor, numerator_bits, denominator_bits); @@ -887,7 +907,7 @@ fn expand_f32_division( let ratio_lt = b.compare( func, - CmpOp::Lt, + CmpOp::Ult, normalized_numerator.significand, normalized_denominator.significand, ); @@ -915,7 +935,7 @@ fn expand_f32_division( for step in 0..27 { let bit = b.compare( func, - CmpOp::Ge, + CmpOp::Uge, remainder, normalized_denominator.significand, ); @@ -932,12 +952,12 @@ fn expand_f32_division( } let hh1 = b.constant(func, 256); - let exp_le = b.compare(func, CmpOp::Le, result_exponent, hh1); + let exp_le = b.compare(func, CmpOp::Ule, result_exponent, hh1); let is_tiny = bool_as_u32(func, b, exp_le); let hh2 = b.constant(func, 260); let raw_tiny_shift = b.binary(func, BinOp::Sub, hh2, result_exponent); let hh3 = b.constant(func, 31); - let shift_gt = b.compare(func, CmpOp::Gt, raw_tiny_shift, hh3); + let shift_gt = b.compare(func, CmpOp::Ugt, raw_tiny_shift, hh3); let shift_too_large = bool_as_u32(func, b, shift_gt); let shift_large_set = flag_set(func, b, shift_too_large); let hh4 = b.constant(func, 31); @@ -948,7 +968,7 @@ fn expand_f32_division( let rounded = round_division_quotient(func, b, quotient, remainder, rounding_shift); let hh6 = b.constant(func, 0x0100_0000); - let carry_ge = b.compare(func, CmpOp::Ge, rounded, hh6); + let carry_ge = b.compare(func, CmpOp::Uge, rounded, hh6); let rounded_carry = bool_as_u32(func, b, carry_ge); let carry_set = flag_set(func, b, rounded_carry); let halved = b.shifted(func, BinOp::Shr, rounded, 1); @@ -962,10 +982,10 @@ fn expand_f32_division( let tiny_set2 = flag_set(func, b, is_tiny); let finite_bits = b.select(func, tiny_set2, rounded, normal_bits); let hh8 = b.constant(func, 511); - let overflow_before = b.compare(func, CmpOp::Ge, result_exponent, hh8); + let overflow_before = b.compare(func, CmpOp::Uge, result_exponent, hh8); let overflow_before = bool_as_u32(func, b, overflow_before); let hh9 = b.constant(func, 255); - let overflow_after = b.compare(func, CmpOp::Ge, normal_exponent, hh9); + let overflow_after = b.compare(func, CmpOp::Uge, normal_exponent, hh9); let overflow_after = bool_as_u32(func, b, overflow_after); let tiny_set3 = flag_set(func, b, is_tiny); let hh10 = b.constant(func, 0); @@ -988,10 +1008,10 @@ fn expand_f32_division( let den_inf_eq = b.equal_constant(func, denominator_magnitude, 0x7f80_0000); let denominator_infinity = bool_as_u32(func, b, den_inf_eq); let hh13 = b.constant(func, 0x7f80_0000); - let num_nan_gt = b.compare(func, CmpOp::Gt, numerator_magnitude, hh13); + let num_nan_gt = b.compare(func, CmpOp::Ugt, numerator_magnitude, hh13); let numerator_nan = bool_as_u32(func, b, num_nan_gt); let hh14 = b.constant(func, 0x7f80_0000); - let den_nan_gt = b.compare(func, CmpOp::Gt, denominator_magnitude, hh14); + let den_nan_gt = b.compare(func, CmpOp::Ugt, denominator_magnitude, hh14); let denominator_nan = bool_as_u32(func, b, den_nan_gt); let di_set = flag_set(func, b, denominator_infinity); let hh15 = b.constant(func, 0); @@ -1105,10 +1125,10 @@ pub fn expand_f32_div(func: &mut Function) -> bool { // --- Mulh -------------------------------------------------------------------- fn is_mulh(func: &Function, inst: Inst) -> bool { - matches!(func.opcode(inst), Opcode::Arith(a) if a.op == BinOp::Mulh) + matches!(func.opcode(inst), Opcode::Arith(a) if matches!(a.op, BinOp::UMulh | BinOp::SMulh)) } -/// Expand the high half of a full-width product into half-width limbs. +/// Expand the high half of a full-width product (`UMulh` and `SMulh`) into half-width limbs. /// Returns whether anything changed. pub fn expand_mulh(func: &mut Function) -> bool { let mut changed = false; @@ -1139,7 +1159,7 @@ pub fn expand_mulh(func: &mut Function) -> bool { continue; }; let ty = func.value_type(result); - let high = emit_mulh_limbs(func, &mut out, a.lhs, a.rhs, ty); + let high = emit_mulh_limbs(func, &mut out, a.lhs, a.rhs, ty, a.op == BinOp::SMulh); func.replace_all_uses(result, high); } func.set_block_insts(block, &out); @@ -1147,20 +1167,21 @@ pub fn expand_mulh(func: &mut Function) -> bool { changed } -/// Emit the limb sequence for `high half of (lhs * rhs)` at type `ty`, -/// appending each instruction to `out`, and return the high-half value. +/// Emit the limb sequence for the high half of `lhs * rhs` at type `ty`, read as unsigned +/// or (`signed`) as two's complement signed, appending each instruction to `out`, and return +/// the high-half value. fn emit_mulh_limbs( func: &mut Function, out: &mut Vec, lhs: Value, rhs: Value, ty: Type, + signed: bool, ) -> Value { - let info = match func.types.type_kind(ty) { - TypeKind::Int(i) => *i, + let w = match func.types.type_kind(ty) { + TypeKind::Int(i) => i.bits, _ => unreachable!("mulh is integer-only"), }; - let w = info.bits; let h = (w / 2) as i64; let mask_h: i64 = (1i64 << (w / 2)) - 1; @@ -1198,13 +1219,15 @@ fn emit_mulh_limbs( let s0 = b.op(BinOp::Add, hihi, lohi_hi); let s1 = b.op(BinOp::Add, s0, hilo_hi); let unsigned_high = b.op(BinOp::Add, s1, cross_hi); - if !info.signed { + if !signed { return unsigned_high; } + // Signed correction: subtract each operand where the other is negative (an arithmetic + // shift by `w - 1` is all ones for a negative operand, zero otherwise). let wm1 = b.konst((w - 1) as i64); - let amask = b.op(BinOp::Shr, lhs, wm1); - let bmask = b.op(BinOp::Shr, rhs, wm1); + let amask = b.op(BinOp::Sar, lhs, wm1); + let bmask = b.op(BinOp::Sar, rhs, wm1); let ca = b.op(BinOp::BitAnd, amask, rhs); let cb = b.op(BinOp::BitAnd, bmask, lhs); let c0 = b.op(BinOp::Sub, unsigned_high, ca); @@ -1240,7 +1263,11 @@ struct Shape { b_elem: Type, acc: Type, elem_bytes: i64, - convert: bool, + /// How a loaded element of A / B widens to the accumulator type; `None` when it already + /// has it. An int8 element is loaded as the raw byte (zero-extended, signless), so its + /// signedness is the choice of `Sext` or `Zext` here. + a_widen: Option, + b_widen: Option, acc_is_float: bool, } @@ -1249,14 +1276,15 @@ const C_ELEM_BYTES: i64 = 4; fn shape_of(func: &mut Function, mm: &MatMul) -> Shape { let f32_t = func.types.intern(TypeKind::Float(FloatKind::F32)); - let i32_t = func.types.intern(int_ty_kind(32, true)); + let i32_t = func.types.intern(int_ty_kind(32)); match mm.dtype { MatMulType::Fp32 => Shape { a_elem: f32_t, b_elem: f32_t, acc: f32_t, elem_bytes: 4, - convert: false, + a_widen: None, + b_widen: None, acc_is_float: true, }, MatMulType::Int8 | MatMulType::Uint8 => { @@ -1268,12 +1296,20 @@ fn shape_of(func: &mut Function, mm: &MatMul) -> Shape { Some(s) => (s.a_unsigned, s.b_unsigned), None => (default_unsigned, default_unsigned), }; + let widen = |unsigned: bool| { + Some(if unsigned { + ConvertKind::Zext + } else { + ConvertKind::Sext + }) + }; Shape { - a_elem: func.types.intern(int_ty_kind(8, !a_unsigned)), - b_elem: func.types.intern(int_ty_kind(8, !b_unsigned)), + a_elem: func.types.intern(int_ty_kind(8)), + b_elem: func.types.intern(int_ty_kind(8)), acc: i32_t, elem_bytes: 1, - convert: true, + a_widen: widen(a_unsigned), + b_widen: widen(b_unsigned), acc_is_float: false, } } @@ -1284,7 +1320,8 @@ fn shape_of(func: &mut Function, mm: &MatMul) -> Shape { b_elem: func.types.intern(TypeKind::Float(FloatKind::F16)), acc: func.types.intern(TypeKind::Float(FloatKind::F32)), elem_bytes: 2, - convert: true, + a_widen: Some(ConvertKind::FpResize), + b_widen: Some(ConvertKind::FpResize), acc_is_float: true, }, } @@ -1367,7 +1404,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { let ptr_t = func.types.ptr_global(); let bool_t = func.types.intern(TypeKind::Bool); - let i32_t = func.types.intern(int_ty_kind(32, true)); + let i32_t = func.types.intern(int_ty_kind(32)); // Snapshot the block's instruction list: `set_block_insts` clears the list // it would then read from, so the two halves must be copied out first. @@ -1412,7 +1449,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { nest.i_head, bool_t, Opcode::Icmp(super::function::Compare { - op: CmpOp::Lt, + op: CmpOp::Slt, lhs: i, rhs: m_bound, }), @@ -1436,7 +1473,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { nest.j_head, bool_t, Opcode::Icmp(super::function::Compare { - op: CmpOp::Lt, + op: CmpOp::Slt, lhs: j, rhs: n_bound, }), @@ -1452,14 +1489,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { // j-header also runs with `j == n`, where `c_elem` is one past the end of // the C row. let acc_init = if mm.accumulate { - func.append_inst( - nest.j_body, - shape.acc, - Opcode::Load(super::function::Load { - ptr: c_elem, - volatile: false, - }), - ) + func.append_load(nest.j_body, shape.acc, c_elem, MemFlags::new()) } else { acc_zero }; @@ -1474,7 +1504,7 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { nest.p_head, bool_t, Opcode::Icmp(super::function::Compare { - op: CmpOp::Lt, + op: CmpOp::Slt, lhs: p, rhs: k_bound, }), @@ -1486,39 +1516,15 @@ fn expand_matmul_site(func: &mut Function, site: &Site) { super::function::EdgeDesc::bare(nest.j_latch), ); - let a_val = func.append_inst( - nest.p_body, - shape.a_elem, - Opcode::Load(super::function::Load { - ptr: a_elem, - volatile: false, - }), - ); - let b_val = func.append_inst( - nest.p_body, - shape.b_elem, - Opcode::Load(super::function::Load { - ptr: b_elem, - volatile: false, - }), - ); - let a_op = if shape.convert { - func.append_inst( - nest.p_body, - shape.acc, - Opcode::Convert(super::function::Convert { value: a_val }), - ) - } else { - a_val + let a_val = func.append_load(nest.p_body, shape.a_elem, a_elem, MemFlags::new()); + let b_val = func.append_load(nest.p_body, shape.b_elem, b_elem, MemFlags::new()); + let a_op = match shape.a_widen { + Some(kind) => func.append_convert(nest.p_body, shape.acc, kind, a_val), + None => a_val, }; - let b_op = if shape.convert { - func.append_inst( - nest.p_body, - shape.acc, - Opcode::Convert(super::function::Convert { value: b_val }), - ) - } else { - b_val + let b_op = match shape.b_widen { + Some(kind) => func.append_convert(nest.p_body, shape.acc, kind, b_val), + None => b_val, }; let product = func.append_inst( nest.p_body, diff --git a/src/ir/src/expand/tests/interp.rs b/src/ir/src/expand/tests/interp.rs new file mode 100644 index 0000000..6d3851a --- /dev/null +++ b/src/ir/src/expand/tests/interp.rs @@ -0,0 +1,326 @@ +//! A small reference interpreter for the IR the expansions produce, used to check their +//! numerics against the host implementations. Values are bit patterns, held zero-extended at +//! their type's width (the backend convention); `Sext`, `Sar`, `SDiv`, ... re-read the bits as +//! signed on demand. + +use alloc::vec::Vec; +use std::collections::HashMap; + +use crate::function::{ + BinOp, Block, CmpOp, ConvertKind, Function, Opcode, Terminator, UnaryOp, Value, +}; +use crate::types::{FloatKind, Type, TypeKind}; + +fn bits_of(func: &Function, ty: Type) -> u32 { + match func.types.type_kind(ty) { + TypeKind::Int(d) => u32::from(d.bits), + TypeKind::Bool => 1, + TypeKind::Ptr(_) => 64, + TypeKind::Float(FloatKind::F16) => 16, + TypeKind::Float(FloatKind::F32) => 32, + TypeKind::Float(FloatKind::F64) => 64, + other => panic!("interpreter: unsupported type {other:?}"), + } +} + +fn mask(bits: u32) -> u64 { + if bits >= 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + } +} + +fn sext(x: u64, bits: u32) -> i64 { + if bits >= 64 { + x as i64 + } else { + ((x << (64 - bits)) as i64) >> (64 - bits) + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Fp { + F16, + F32, + F64, +} + +fn fp_kind(func: &Function, ty: Type) -> Option { + match func.types.type_kind(ty) { + TypeKind::Float(FloatKind::F16) => Some(Fp::F16), + TypeKind::Float(FloatKind::F32) => Some(Fp::F32), + TypeKind::Float(FloatKind::F64) => Some(Fp::F64), + TypeKind::Float(FloatKind::F128) => panic!("interpreter: f128"), + _ => None, + } +} + +fn f16_to_f64(bits: u64) -> f64 { + let sign = if bits & 0x8000 != 0 { -1.0 } else { 1.0 }; + let exp = ((bits >> 10) & 0x1f) as i32; + let frac = (bits & 0x3ff) as f64; + match exp { + 0 => sign * frac * 2f64.powi(-24), + 31 if frac == 0.0 => sign * f64::INFINITY, + 31 => f64::NAN, + _ => sign * (1.0 + frac / 1024.0) * 2f64.powi(exp - 15), + } +} + +/// The value of an IEEE half-precision bit pattern, exactly (every half is an f32). +pub fn f16_to_f32(bits: u16) -> f32 { + f16_to_f64(u64::from(bits)) as f32 +} + +fn to_f64(kind: Fp, bits: u64) -> f64 { + match kind { + Fp::F16 => f16_to_f64(bits), + Fp::F32 => f64::from(f32::from_bits(bits as u32)), + Fp::F64 => f64::from_bits(bits), + } +} + +fn from_f64(kind: Fp, v: f64) -> u64 { + match kind { + Fp::F16 => panic!("interpreter: narrowing to f16"), + Fp::F32 => u64::from((v as f32).to_bits()), + Fp::F64 => v.to_bits(), + } +} + +fn int_arith(op: BinOp, l: u64, r: u64, bits: u32) -> u64 { + let m = mask(bits); + let (sl, sr) = (sext(l, bits), sext(r, bits)); + let raw = match op { + BinOp::Add => l.wrapping_add(r), + BinOp::Sub => l.wrapping_sub(r), + BinOp::Mul => l.wrapping_mul(r), + BinOp::BitAnd => l & r, + BinOp::BitOr => l | r, + BinOp::BitXor => l ^ r, + BinOp::Shl => l.checked_shl(r as u32).unwrap_or(0), + BinOp::Shr => l.checked_shr(r as u32).unwrap_or(0), + BinOp::Sar => (sl >> r.min(63)) as u64, + BinOp::UDiv => l / r, + BinOp::URem => l % r, + BinOp::SDiv => sl.wrapping_div(sr) as u64, + BinOp::SRem => sl.wrapping_rem(sr) as u64, + BinOp::UMulh => ((u128::from(l) * u128::from(r)) >> bits) as u64, + BinOp::SMulh => ((i128::from(sl) * i128::from(sr)) >> bits) as u64, + BinOp::Div | BinOp::Rem => panic!("interpreter: {op:?} on an integer"), + }; + raw & m +} + +fn float_arith(op: BinOp, kind: Fp, l: u64, r: u64) -> u64 { + let (l, r) = (to_f64(kind, l), to_f64(kind, r)); + let v = match op { + BinOp::Add => l + r, + BinOp::Sub => l - r, + BinOp::Mul => l * r, + BinOp::Div => l / r, + BinOp::Rem => l % r, + other => panic!("interpreter: {other:?} on a float"), + }; + from_f64(kind, v) +} + +fn compare(op: CmpOp, kind: Option, l: u64, r: u64, bits: u32) -> bool { + if let Some(kind) = kind { + let (l, r) = (to_f64(kind, l), to_f64(kind, r)); + return match op { + CmpOp::Eq => l == r, + CmpOp::Ne => l != r, + CmpOp::Lt => l < r, + CmpOp::Le => l <= r, + CmpOp::Gt => l > r, + CmpOp::Ge => l >= r, + other => panic!("interpreter: {other:?} on a float"), + }; + } + let (sl, sr) = (sext(l, bits), sext(r, bits)); + match op { + CmpOp::Eq => l == r, + CmpOp::Ne => l != r, + CmpOp::Slt => sl < sr, + CmpOp::Sle => sl <= sr, + CmpOp::Sgt => sl > sr, + CmpOp::Sge => sl >= sr, + CmpOp::Ult => l < r, + CmpOp::Ule => l <= r, + CmpOp::Ugt => l > r, + CmpOp::Uge => l >= r, + other => panic!("interpreter: {other:?} on an integer"), + } +} + +/// Runs functions over a flat little-endian memory. +pub struct Interp { + pub mem: Vec, +} + +impl Interp { + pub fn new(mem: Vec) -> Self { + Self { mem } + } + + fn read(&self, addr: u64, bytes: u32) -> u64 { + (0..bytes as usize).fold(0u64, |acc, i| { + acc | u64::from(self.mem[addr as usize + i]) << (8 * i) + }) + } + + fn write(&mut self, addr: u64, bytes: u32, value: u64) { + for i in 0..bytes as usize { + self.mem[addr as usize + i] = (value >> (8 * i)) as u8; + } + } + + /// Executes `func` from its entry block with `args` bound to the entry parameters and + /// returns the returned values. + pub fn run(&mut self, func: &Function, args: &[u64]) -> Vec { + let mut vals: HashMap = HashMap::new(); + let mut block = Block::from_u32(0); + let entry_params = func.block_params(block).to_vec(); + assert_eq!(entry_params.len(), args.len()); + for (p, a) in entry_params.iter().zip(args) { + vals.insert(*p, *a & mask(bits_of(func, func.value_type(*p)))); + } + let mut steps = 0u64; + 'blocks: loop { + for &inst in func.block_insts(block) { + steps += 1; + assert!(steps < 5_000_000, "interpreter: runaway loop"); + let result = func.inst_result(inst); + let result_ty = result.map(|r| func.value_type(r)); + let out: Option = match func.opcode(inst) { + Opcode::Iconst(c) => Some(c as u64 & mask(bits_of(func, result_ty.unwrap()))), + Opcode::Fconst(f) => Some(match fp_kind(func, result_ty.unwrap()) { + Some(Fp::F32) => u64::from((f as f32).to_bits()), + Some(Fp::F64) => f.to_bits(), + other => panic!("interpreter: fconst {other:?}"), + }), + Opcode::Arith(a) => { + let ty = func.value_type(a.lhs); + let (l, r) = (vals[&a.lhs], vals[&a.rhs]); + Some(match fp_kind(func, ty) { + Some(kind) => float_arith(a.op, kind, l, r), + None => int_arith(a.op, l, r, bits_of(func, ty)), + }) + } + Opcode::ArithImm(a) => { + let ty = func.value_type(a.lhs); + let bits = bits_of(func, ty); + Some(int_arith( + a.op, + vals[&a.lhs], + a.imm as u64 & mask(bits), + bits, + )) + } + Opcode::Icmp(c) => { + let ty = func.value_type(c.lhs); + Some(u64::from(compare( + c.op, + fp_kind(func, ty), + vals[&c.lhs], + vals[&c.rhs], + bits_of(func, ty), + ))) + } + Opcode::Select(s) => Some(if vals[&s.cond] != 0 { + vals[&s.then] + } else { + vals[&s.else_] + }), + Opcode::Convert(cv) => { + let (src_ty, dst_ty) = (func.value_type(cv.value), result_ty.unwrap()); + let (sb, db) = (bits_of(func, src_ty), bits_of(func, dst_ty)); + let v = vals[&cv.value]; + Some(match cv.kind { + ConvertKind::Trunc => { + assert!(db < sb); + v & mask(db) + } + ConvertKind::Zext => { + assert!(db > sb); + v + } + ConvertKind::Sext => { + assert!(db > sb); + sext(v, sb) as u64 & mask(db) + } + ConvertKind::SiToFp => { + from_f64(fp_kind(func, dst_ty).unwrap(), sext(v, sb) as f64) + } + ConvertKind::UiToFp => { + from_f64(fp_kind(func, dst_ty).unwrap(), v as f64) + } + ConvertKind::FpToSi => { + let f = to_f64(fp_kind(func, src_ty).unwrap(), v); + let (lo, hi) = if db >= 64 { + (i64::MIN, i64::MAX) + } else { + (-(1i64 << (db - 1)), (1i64 << (db - 1)) - 1) + }; + (f as i64).clamp(lo, hi) as u64 & mask(db) + } + ConvertKind::FpToUi => { + let f = to_f64(fp_kind(func, src_ty).unwrap(), v); + (f as u64).min(mask(db)) + } + ConvertKind::FpResize => { + let f = to_f64(fp_kind(func, src_ty).unwrap(), v); + from_f64(fp_kind(func, dst_ty).unwrap(), f) + } + }) + } + Opcode::Unary(u) => { + assert_eq!(u.op, UnaryOp::Reinterpret); + let bits = bits_of(func, result_ty.unwrap()); + assert_eq!(bits, bits_of(func, func.value_type(u.value))); + Some(vals[&u.value] & mask(bits)) + } + Opcode::Load(l) => { + let bytes = bits_of(func, result_ty.unwrap()) / 8; + Some(self.read(vals[&l.ptr], bytes)) + } + Opcode::Store(st) => { + let bytes = bits_of(func, func.value_type(st.value)) / 8; + self.write(vals[&st.ptr], bytes, vals[&st.value]); + None + } + Opcode::If(cf) => { + let jump = if vals[&cf.cond] != 0 { + cf.then + } else { + cf.else_ + }; + let moved: Vec = + func.block_args(jump).iter().map(|a| vals[a]).collect(); + for (p, v) in func.block_params(jump.target).iter().zip(moved) { + vals.insert(*p, v); + } + block = jump.target; + continue 'blocks; + } + other => panic!("interpreter: unsupported opcode {other:?}"), + }; + if let (Some(r), Some(v)) = (result, out) { + vals.insert(r, v); + } + } + match func.terminator(block).expect("terminated block") { + Terminator::Ret(r) => return r.slice().iter().map(|v| vals[v]).collect(), + Terminator::Jump(j) => { + let moved: Vec = func.block_args(j).iter().map(|a| vals[a]).collect(); + for (p, v) in func.block_params(j.target).iter().zip(moved) { + vals.insert(*p, v); + } + block = j.target; + } + } + } + } +} diff --git a/src/ir/src/expand/tests/mod.rs b/src/ir/src/expand/tests/mod.rs index 3edc19d..2416e62 100644 --- a/src/ir/src/expand/tests/mod.rs +++ b/src/ir/src/expand/tests/mod.rs @@ -1,2 +1,3 @@ +mod interp; mod tests_matmul; mod tests_scalar; diff --git a/src/ir/src/expand/tests/tests_matmul.rs b/src/ir/src/expand/tests/tests_matmul.rs index a4ce189..dd47a10 100644 --- a/src/ir/src/expand/tests/tests_matmul.rs +++ b/src/ir/src/expand/tests/tests_matmul.rs @@ -2,13 +2,15 @@ use alloc::string::ToString; +use super::interp::{Interp, f16_to_f32}; use crate::attribute::Attribute; use crate::expand::expand_matmul; use crate::function::{ - Arith, AttrTarget, Function, MatMulQuant, MatMulQuantOut, MatMulScale, MatMulType, Opcode, Ret, - Terminator, + Arith, AttrTarget, ConvertKind, Function, InputSigns, MatMulQuant, MatMulQuantOut, MatMulScale, + MatMulType, Opcode, Ret, Terminator, }; use crate::types::{FloatKind, Type, TypeKind}; +use crate::verify::{Profile, verify}; use alloc::{vec, vec::Vec}; fn f32_ty(func: &mut Function) -> Type { @@ -173,3 +175,213 @@ fn expand_matmul_skips_block_keyed_attributes() { ); assert!(!expand_matmul(&mut func)); } + +/// Builds `fn() { matmul c, a, b [m x n x k] }` over the flat memory layout A, B, C (in that +/// order, `a_len` and `b_len` bytes long), expands it, checks it still verifies and runs it +/// on `mem`, returning the final memory. +#[allow(clippy::too_many_arguments)] +fn run_matmul( + dtype: MatMulType, + (m, n, k): (u16, u16, u16), + signs: Option, + accumulate: bool, + (a_len, b_len): (usize, usize), + mem: Vec, +) -> (Function, Vec) { + let mut func = Function::new(); + let ptr_t = func.types.ptr_global(); + let block = func.append_block(); + let a = func.append_inst(block, ptr_t, Opcode::Iconst(0)); + let b = func.append_inst(block, ptr_t, Opcode::Iconst(a_len as i64)); + let c = func.append_inst(block, ptr_t, Opcode::Iconst((a_len + b_len) as i64)); + func.append_matmul_embedded(block, a, b, c, m, n, k, dtype, accumulate, signs); + func.set_terminator(block, Terminator::Ret(Ret::none())); + assert!(expand_matmul(&mut func)); + let diags = verify(&func, Profile::High); + assert!(diags.ok(), "{:?}", diags.items()); + let mut interp = Interp::new(mem); + interp.run(&func, &[]); + (func, interp.mem) +} + +fn pseudo_random_bytes(len: usize, mut seed: u64) -> Vec { + (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed >> 24) as u8 + }) + .collect() +} + +/// Runs an int8 matmul and compares C with the plain host computation, reading each operand +/// as unsigned or signed as `a_unsigned` / `b_unsigned` say. +fn check_int8( + dtype: MatMulType, + signs: Option, + a_unsigned: bool, + b_unsigned: bool, + accumulate: bool, +) { + let (m, n, k) = (3usize, 4usize, 5usize); + let a = pseudo_random_bytes(m * k, 0x9e37_79b9_7f4a_7c15); + let b = pseudo_random_bytes(k * n, 0x1234_5678_9abc_def1); + let c_init = pseudo_random_bytes(m * n * 4, 0x0fed_cba9_8765_4321); + let mut mem = a.clone(); + mem.extend_from_slice(&b); + mem.extend_from_slice(&c_init); + let (func, out) = run_matmul( + dtype, + (m as u16, n as u16, k as u16), + signs, + accumulate, + (a.len(), b.len()), + mem, + ); + let widen = |x: u8, unsigned: bool| { + if unsigned { + i32::from(x) + } else { + i32::from(x as i8) + } + }; + for i in 0..m { + for j in 0..n { + let at = 4 * (i * n + j); + let mut want = if accumulate { + i32::from_le_bytes(c_init[at..at + 4].try_into().unwrap()) + } else { + 0 + }; + for p in 0..k { + want = want.wrapping_add( + widen(a[i * k + p], a_unsigned) * widen(b[p * n + j], b_unsigned), + ); + } + let base = a.len() + b.len() + at; + let got = i32::from_le_bytes(out[base..base + 4].try_into().unwrap()); + assert_eq!( + got, want, + "{dtype:?} {signs:?} acc={accumulate} c[{i}][{j}]" + ); + } + } + // The widening is the explicit part of the expansion: one `Sext` or `Zext` per operand. + let widens = |kind: ConvertKind| { + count( + &func, + |op| matches!(op, Opcode::Convert(c) if c.kind == kind), + ) + }; + let (zexts, sexts) = (widens(ConvertKind::Zext), widens(ConvertKind::Sext)); + assert_eq!(zexts, usize::from(a_unsigned) + usize::from(b_unsigned)); + assert_eq!(sexts, usize::from(!a_unsigned) + usize::from(!b_unsigned)); +} + +#[test] +fn expand_matmul_int8_reads_both_operands_signed() { + check_int8(MatMulType::Int8, None, false, false, false); + check_int8(MatMulType::Int8, None, false, false, true); +} + +#[test] +fn expand_matmul_uint8_reads_both_operands_unsigned() { + check_int8(MatMulType::Uint8, None, true, true, false); + check_int8(MatMulType::Uint8, None, true, true, true); +} + +#[test] +fn expand_matmul_input_signs_override_the_dtype_per_operand() { + let mixed = Some(InputSigns { + a_unsigned: true, + b_unsigned: false, + }); + check_int8(MatMulType::Int8, mixed, true, false, false); + check_int8(MatMulType::Uint8, mixed, true, false, true); + let other = Some(InputSigns { + a_unsigned: false, + b_unsigned: true, + }); + check_int8(MatMulType::Uint8, other, false, true, false); + check_int8(MatMulType::Int8, other, false, true, true); +} + +#[test] +fn expand_matmul_fp16_widens_with_fp_resize() { + let (m, n, k) = (2usize, 3usize, 4usize); + // Finite halves, positive and negative: 1.0, -2.5, 0.375, 100.0, -0.0625, ... + let halves: [u16; 8] = [ + 0x3c00, 0xc100, 0x3600, 0x5640, 0xac00, 0x4200, 0xbd00, 0x0400, + ]; + let pick = |i: usize| halves[i % halves.len()]; + let a: Vec = (0..m * k).map(|i| pick(i * 3 + 1)).collect(); + let b: Vec = (0..k * n).map(|i| pick(i * 5 + 2)).collect(); + let mut mem: Vec = Vec::new(); + for h in a.iter().chain(&b) { + mem.extend_from_slice(&h.to_le_bytes()); + } + let (a_len, b_len) = (a.len() * 2, b.len() * 2); + mem.extend(core::iter::repeat_n(0u8, m * n * 4)); + let (func, out) = run_matmul( + MatMulType::Fp16, + (m as u16, n as u16, k as u16), + None, + false, + (a_len, b_len), + mem, + ); + for i in 0..m { + for j in 0..n { + let mut want = 0.0f32; + for p in 0..k { + want += f16_to_f32(a[i * k + p]) * f16_to_f32(b[p * n + j]); + } + let base = a_len + b_len + 4 * (i * n + j); + let got = f32::from_le_bytes(out[base..base + 4].try_into().unwrap()); + assert_eq!(got.to_bits(), want.to_bits(), "c[{i}][{j}]"); + } + } + assert_eq!( + count( + &func, + |op| matches!(op, Opcode::Convert(c) if c.kind == ConvertKind::FpResize) + ), + 2 + ); +} + +#[test] +fn expand_matmul_fp32_accumulates_in_k_order() { + let (m, n, k) = (2usize, 2usize, 3usize); + let a = [1.5f32, -2.0, 0.25, 3.0, 0.5, -1.0]; + let b = [2.0f32, 4.0, -1.0, 0.5, 8.0, 3.0]; + let mut mem: Vec = Vec::new(); + for x in a.iter().chain(&b) { + mem.extend_from_slice(&x.to_le_bytes()); + } + let c_init = [10.0f32, 20.0, 30.0, 40.0]; + for x in &c_init { + mem.extend_from_slice(&x.to_le_bytes()); + } + let (func, out) = run_matmul( + MatMulType::Fp32, + (m as u16, n as u16, k as u16), + None, + true, + (a.len() * 4, b.len() * 4), + mem, + ); + for i in 0..m { + for j in 0..n { + let mut want = c_init[i * n + j]; + for p in 0..k { + want += a[i * k + p] * b[p * n + j]; + } + let base = (a.len() + b.len()) * 4 + 4 * (i * n + j); + let got = f32::from_le_bytes(out[base..base + 4].try_into().unwrap()); + assert_eq!(got.to_bits(), want.to_bits(), "c[{i}][{j}]"); + } + } + assert_eq!(count(&func, |op| matches!(op, Opcode::Convert(_))), 0); +} diff --git a/src/ir/src/expand/tests/tests_scalar.rs b/src/ir/src/expand/tests/tests_scalar.rs index 2e74c27..5c8cf8c 100644 --- a/src/ir/src/expand/tests/tests_scalar.rs +++ b/src/ir/src/expand/tests/tests_scalar.rs @@ -4,34 +4,40 @@ use alloc::string::ToString; use alloc::vec::Vec; use super::super::*; +use super::interp::Interp; use crate::attribute::{AttrValue, Attribute, Custom}; -use crate::expand::{expand_low_float, expand_mulh, expand_nv_fp4}; +use crate::expand::{expand_f32_div, expand_low_float, expand_mulh, expand_nv_fp4}; use crate::function::AttrTarget; use crate::function::Function; use crate::function::{Arith, Ret, Terminator, Unary, UnaryOp}; +use crate::low_float::{self, Format}; +use crate::nvfp4::{self, ScaleApplication}; use crate::types::{IntDesc, Type, TypeKind}; +use crate::verify::{Profile, verify}; -fn unsigned_int(func: &mut Function, bits: u16) -> Type { - func.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits, - })) +fn int_ty(func: &mut Function, bits: u16) -> Type { + func.types.intern(TypeKind::Int(IntDesc { bits })) +} + +fn assert_verifies(func: &Function) { + let diags = verify(func, Profile::High); + assert!(diags.ok(), "{:?}", diags.items()); } fn f32_ty(func: &mut Function) -> Type { func.types.intern(TypeKind::Float(FloatKind::F32)) } -fn u8_ty(func: &mut Function) -> Type { - unsigned_int(func, 8) +fn i8_ty(func: &mut Function) -> Type { + int_ty(func, 8) } -fn u16_ty(func: &mut Function) -> Type { - unsigned_int(func, 16) +fn i16_ty(func: &mut Function) -> Type { + int_ty(func, 16) } -fn u32_ty(func: &mut Function) -> Type { - unsigned_int(func, 32) +fn i32_ty(func: &mut Function) -> Type { + int_ty(func, 32) } fn custom_attr(key: &str) -> Attribute { @@ -48,8 +54,8 @@ fn custom_attr(key: &str) -> Attribute { fn expand_low_float_no_op_without_trigger() { let mut func = Function::new(); let block = func.append_block(); - let u32t = u32_ty(&mut func); - let v = func.append_inst(block, u32t, Opcode::Iconst(41)); + let i32t = i32_ty(&mut func); + let v = func.append_inst(block, i32t, Opcode::Iconst(41)); let inst = func.defining_inst(v).unwrap(); func.add_attr(AttrTarget::Value(v), custom_attr("unchanged")); func.set_terminator(block, Terminator::Ret(Ret::one(v))); @@ -64,7 +70,7 @@ fn expand_low_float_no_op_without_trigger() { #[test] fn expand_low_float_rewrites_and_migrates_attrs() { let mut func = Function::new(); - let payload_t = u16_ty(&mut func); + let payload_t = i16_ty(&mut func); let f32t = f32_ty(&mut func); let block = func.append_block(); let payload = func.append_block_param(block, payload_t); @@ -125,12 +131,12 @@ fn expand_low_float_rewrites_and_migrates_attrs() { fn expand_low_float_encode_fp8_moves_attrs_to_f32_boundary() { let mut func = Function::new(); let f32t = f32_ty(&mut func); - let u8t = u8_ty(&mut func); + let i8t = i8_ty(&mut func); let block = func.append_block(); let source = func.append_block_param(block, f32t); let encoded = func.append_inst( block, - u8t, + i8t, Opcode::EncodeLowFloat(crate::function::LowFloatConvert { value: source, format: crate::low_float::Format::F8E5M2, @@ -171,11 +177,11 @@ fn expand_low_float_encode_fp8_moves_attrs_to_f32_boundary() { // --- NVFP4 ------------------------------------------------------------------- fn nvfp4_fixture(func: &mut Function, dequant: bool) -> (Block, Value) { - let u8t = u8_ty(func); + let i8t = i8_ty(func); let f32t = f32_ty(func); let block = func.append_block(); - let payload = func.append_block_param(block, u8t); - let block_scale = func.append_block_param(block, u8t); + let payload = func.append_block_param(block, i8t); + let block_scale = func.append_block_param(block, i8t); let value = func.append_block_param(block, f32t); let global_scale = func.append_block_param(block, f32t); let conv = crate::function::NvFp4Convert { @@ -188,7 +194,7 @@ fn nvfp4_fixture(func: &mut Function, dequant: bool) -> (Block, Value) { let result = if dequant { func.append_inst(block, f32t, Opcode::DequantizeNvfp4(conv)) } else { - func.append_inst(block, u8t, Opcode::QuantizeNvfp4(conv)) + func.append_inst(block, i8t, Opcode::QuantizeNvfp4(conv)) }; func.set_terminator(block, Terminator::Ret(Ret::one(result))); (block, result) @@ -198,8 +204,8 @@ fn nvfp4_fixture(func: &mut Function, dequant: bool) -> (Block, Value) { fn expand_nv_fp4_no_op_without_trigger() { let mut func = Function::new(); let block = func.append_block(); - let u32t = u32_ty(&mut func); - let v = func.append_inst(block, u32t, Opcode::Iconst(43)); + let i32t = i32_ty(&mut func); + let v = func.append_inst(block, i32t, Opcode::Iconst(43)); func.set_terminator(block, Terminator::Ret(Ret::one(v))); assert!(!expand_nv_fp4(&mut func)); assert!(matches!( @@ -318,45 +324,40 @@ fn expand_nv_fp4_migrates_attrs_to_semantic_boundary() { // --- Mulh -------------------------------------------------------------------- -fn mulh_const_fixture(func: &mut Function, bits: u16, signed: bool) -> (Block, Inst) { - let ty = func.types.intern(TypeKind::Int(IntDesc { signed, bits })); +fn mulh_const_fixture(func: &mut Function, bits: u16, op: BinOp) -> (Block, Inst) { + let ty = int_ty(func, bits); let block = func.append_block(); let a = func.append_inst(block, ty, Opcode::Iconst(0x1234_5678_9abcu64 as i64)); let b = func.append_inst(block, ty, Opcode::Iconst(0xfeed_beefu64 as i64)); - let r = func.append_inst( - block, - ty, - Opcode::Arith(Arith { - op: BinOp::Mulh, - lhs: a, - rhs: b, - }), - ); + let r = func.append_inst(block, ty, Opcode::Arith(Arith { op, lhs: a, rhs: b })); func.set_terminator(block, Terminator::Ret(Ret::one(r))); (block, func.defining_inst(r).unwrap()) } #[test] fn expand_mulh_no_mulh_survives_and_idempotent() { - let mut func = Function::new(); - let (block, _) = mulh_const_fixture(&mut func, 64, true); - assert!(expand_mulh(&mut func)); - assert!(!expand_mulh(&mut func)); - for inst in func.block_insts(block).to_vec() { - assert!(!matches!( - func.opcode(inst), - Opcode::Arith(Arith { - op: BinOp::Mulh, - .. - }) - )); + for op in [BinOp::UMulh, BinOp::SMulh] { + let mut func = Function::new(); + let (block, _) = mulh_const_fixture(&mut func, 64, op); + assert!(expand_mulh(&mut func)); + assert!(!expand_mulh(&mut func)); + for inst in func.block_insts(block).to_vec() { + assert!(!matches!( + func.opcode(inst), + Opcode::Arith(Arith { + op: BinOp::UMulh | BinOp::SMulh, + .. + }) + )); + } + assert_verifies(&func); } } #[test] fn expand_mulh_false_without_trigger() { let mut func = Function::new(); - let ty = u32_ty(&mut func); + let ty = i32_ty(&mut func); let block = func.append_block(); let a = func.append_inst(block, ty, Opcode::Iconst(3)); let b = func.append_inst(block, ty, Opcode::Iconst(7)); @@ -373,124 +374,351 @@ fn expand_mulh_false_without_trigger() { assert!(!expand_mulh(&mut func)); } -#[test] -fn expand_mulh_limb_math_matches_oracle_signed() { +/// The value `fn() { ret }` returns, computed by the expanded limb math. +fn expanded_mulh(bits: u16, op: BinOp, lhs: u64, rhs: u64) -> u64 { let mut func = Function::new(); - let (block, _) = mulh_const_fixture(&mut func, 64, true); + let ty = int_ty(&mut func, bits); + let block = func.append_block(); + let a = func.append_inst(block, ty, Opcode::Iconst(lhs as i64)); + let b = func.append_inst(block, ty, Opcode::Iconst(rhs as i64)); + let r = func.append_inst(block, ty, Opcode::Arith(Arith { op, lhs: a, rhs: b })); + func.set_terminator(block, Terminator::Ret(Ret::one(r))); assert!(expand_mulh(&mut func)); - // Evaluate constant dataflow at declared width: high = (a*b as i128) >> 64 - let ret_v = match func.terminator(block) { - Some(Terminator::Ret(r)) => r.values[0], - _ => panic!("ret"), - }; - let got = eval_const(&func, ret_v); + assert_verifies(&func); + Interp::new(Vec::new()).run(&func, &[])[0] +} + +#[test] +fn expand_mulh_limb_math_matches_oracle_signed() { let a = 0x1234_5678_9abcu64 as i64; let b = 0xfeed_beefu64 as i64; let oracle = ((a as i128) * (b as i128)) >> 64; - assert_eq!(got, oracle as i64); + assert_eq!( + expanded_mulh(64, BinOp::SMulh, a as u64, b as u64), + oracle as u64 + ); } #[test] fn expand_mulh_limb_math_matches_oracle_unsigned() { - let mut func = Function::new(); - let (block, _) = mulh_const_fixture(&mut func, 64, false); - assert!(expand_mulh(&mut func)); - let ret_v = match func.terminator(block) { - Some(Terminator::Ret(r)) => r.values[0], - _ => panic!("ret"), - }; - let got = eval_const(&func, ret_v) as u64; let a = 0x1234_5678_9abcu64; let b = 0xfeed_beefu64; let oracle = ((a as u128) * (b as u128)) >> 64; - assert_eq!(got, oracle as u64); + assert_eq!(expanded_mulh(64, BinOp::UMulh, a, b), oracle as u64); } #[test] fn expand_mulh_works_at_32_bit_width() { - let mut func = Function::new(); - let ty = signed_int32(&mut func); - let block = func.append_block(); - let a = func.append_inst(block, ty, Opcode::Iconst(100_000)); - let b = func.append_inst(block, ty, Opcode::Iconst(100_000)); - let r = func.append_inst( - block, - ty, - Opcode::Arith(Arith { - op: BinOp::Mulh, - lhs: a, - rhs: b, - }), - ); - func.set_terminator(block, Terminator::Ret(Ret::one(r))); - assert!(expand_mulh(&mut func)); - let ret_v = match func.terminator(block) { - Some(Terminator::Ret(r)) => r.values[0], - _ => panic!("ret"), - }; - let got = eval_const(&func, ret_v); let oracle = ((100_000i128) * (100_000i128)) >> 32; - assert_eq!(got, oracle as i64); -} - -fn signed_int32(func: &mut Function) -> Type { - func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })) + assert_eq!( + expanded_mulh(32, BinOp::SMulh, 100_000, 100_000), + oracle as u64 + ); + assert_eq!( + expanded_mulh(32, BinOp::UMulh, 100_000, 100_000), + oracle as u64 + ); } -/// Evaluate a constants-only dataflow at the result type's width. -fn eval_const(func: &Function, v: Value) -> i64 { - let info = match func.types.type_kind(func.value_type(v)) { - TypeKind::Int(i) => *i, - _ => panic!("int expected"), - }; - let wrap = |x: i64| -> i64 { - let bits = info.bits; - if bits >= 64 { - return x; - } - let mask: u64 = if bits >= 64 { !0 } else { (1u64 << bits) - 1 }; - let low = (x as u64) & mask; - if info.signed { - let sign = 1u64 << (bits - 1); - if low & sign != 0 { - (low | !mask) as i64 +#[test] +fn expand_mulh_matches_the_oracle_at_every_width_for_both_signs() { + let samples: [u64; 9] = [ + 0, + 1, + 2, + 0x7f, + 0x80, + 0xff, + 0x8000_0001, + 0x1234_5678_9abc_def0, + u64::MAX, + ]; + for bits in [8u16, 16, 32, 64] { + let mask = if bits == 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + }; + let signed = |x: u64| { + if bits == 64 { + x as i64 } else { - low as i64 + ((x << (64 - bits)) as i64) >> (64 - bits) + } + }; + for &a in &samples { + for &b in &samples { + let (a, b) = (a & mask, b & mask); + let unsigned_want = ((u128::from(a) * u128::from(b)) >> bits) as u64 & mask; + let signed_want = + ((i128::from(signed(a)) * i128::from(signed(b))) >> bits) as u64 & mask; + assert_eq!( + expanded_mulh(bits, BinOp::UMulh, a, b), + unsigned_want, + "umulh i{bits} {a:#x} {b:#x}" + ); + assert_eq!( + expanded_mulh(bits, BinOp::SMulh, a, b), + signed_want, + "smulh i{bits} {a:#x} {b:#x}" + ); } + } + } +} + +// --- Numerics of the scalar expansions ---------------------------------------- + +/// Equal bit patterns, or both NaN (the expansions need not preserve a NaN payload). +fn same_f32(got: u32, want: u32) -> bool { + got == want || (f32::from_bits(got).is_nan() && f32::from_bits(want).is_nan()) +} + +fn sample_floats() -> Vec { + let mut v = alloc::vec![ + 0.0, + -0.0, + 1.0, + -1.0, + 1.5, + -2.5, + 0.1, + -0.1, + 3.7, + 1e-3, + 1e-6, + 1e-10, + 0.001_953_125, + 0.000_976_562_5, + 6.0, + 5.0, + 0.75, + 0.25, + 448.0, + 449.0, + 464.0, + 57344.0, + 61440.0, + 65504.0, + 1e10, + -1e10, + f32::MAX, + f32::MIN_POSITIVE, + f32::from_bits(1), + f32::from_bits(0x007f_ffff), + f32::INFINITY, + f32::NEG_INFINITY, + f32::NAN, + ]; + let mut s = 0x1234_5678_9abc_def0u64; + for _ in 0..1500 { + s ^= s << 13; + s ^= s >> 7; + s ^= s << 17; + v.push(f32::from_bits(s as u32)); + // Values near 1 exercise the rounding paths of the narrow formats. + v.push(f32::from_bits(0x3f00_0000 + (s >> 40) as u32 % 0x0100_0000)); + } + v +} + +#[test] +fn expanded_low_float_decode_matches_the_host_decoder() { + for format in [Format::Bf16, Format::F8E4M3, Format::F8E5M2] { + let mut func = Function::new(); + let payload_t = int_ty(&mut func, format.payload_bits()); + let f32t = f32_ty(&mut func); + let block = func.append_block(); + let payload = func.append_block_param(block, payload_t); + let decoded = func.append_inst( + block, + f32t, + Opcode::DecodeLowFloat(crate::function::LowFloatConvert { + value: payload, + format, + }), + ); + func.set_terminator(block, Terminator::Ret(Ret::one(decoded))); + assert!(expand_low_float(&mut func)); + assert_verifies(&func); + let payloads: Vec = if format == Format::Bf16 { + alloc::vec![ + 0, 1, 0x3f80, 0x8000, 0x7f80, 0xff80, 0x7fc0, 0xc2f7, 0x0001, 0x007f, 0x8001, + 0xffff + ] } else { - low as i64 + (0..=255).collect() + }; + for p in payloads { + let got = Interp::new(Vec::new()).run(&func, &[u64::from(p)])[0] as u32; + let want = low_float::decode(format, p).unwrap().to_bits(); + assert!( + same_f32(got, want), + "{format:?} {p:#x}: {got:#x} != {want:#x}" + ); + } + } +} + +#[test] +fn expanded_low_float_encode_matches_the_host_encoder() { + for format in [Format::Bf16, Format::F8E4M3, Format::F8E5M2] { + let mut func = Function::new(); + let payload_t = int_ty(&mut func, format.payload_bits()); + let f32t = f32_ty(&mut func); + let block = func.append_block(); + let value = func.append_block_param(block, f32t); + let encoded = func.append_inst( + block, + payload_t, + Opcode::EncodeLowFloat(crate::function::LowFloatConvert { value, format }), + ); + func.set_terminator(block, Terminator::Ret(Ret::one(encoded))); + assert!(expand_low_float(&mut func)); + assert_verifies(&func); + for v in sample_floats() { + let got = Interp::new(Vec::new()).run(&func, &[u64::from(v.to_bits())])[0]; + let want = u64::from(low_float::encode(format, v)); + assert_eq!(got, want, "{format:?} {v:e} ({:#x})", v.to_bits()); } + } +} + +const APPLICATIONS: [ScaleApplication; 2] = [ScaleApplication::Multiply, ScaleApplication::Divide]; + +fn nvfp4_function( + dequant: bool, + block_application: ScaleApplication, + global_application: ScaleApplication, +) -> Function { + let mut func = Function::new(); + let i8t = i8_ty(&mut func); + let f32t = f32_ty(&mut func); + let block = func.append_block(); + let payload = func.append_block_param(block, i8t); + let value = func.append_block_param(block, f32t); + let block_scale = func.append_block_param(block, i8t); + let global_scale = func.append_block_param(block, f32t); + let conv = crate::function::NvFp4Convert { + value: if dequant { payload } else { value }, + block_scale, + global_scale, + block_application, + global_application, }; - let inst = func.defining_inst(v).unwrap(); - match func.opcode(inst) { - Opcode::Iconst(c) => wrap(c), - Opcode::Arith(a) => { - let lhs = eval_const(func, a.lhs); - let rhs = eval_const(func, a.rhs); - let raw: i64 = match a.op { - BinOp::Add => lhs.wrapping_add(rhs), - BinOp::Sub => lhs.wrapping_sub(rhs), - BinOp::Mul => lhs.wrapping_mul(rhs), - BinOp::BitAnd => lhs & rhs, - BinOp::BitOr => lhs | rhs, - BinOp::Shr => lhs.wrapping_shr(rhs as u32), - BinOp::Shl => lhs.wrapping_shl(rhs as u32), - _ => panic!("unexpected op"), - }; - wrap(raw) + let result = if dequant { + func.append_inst(block, f32t, Opcode::DequantizeNvfp4(conv)) + } else { + func.append_inst(block, i8t, Opcode::QuantizeNvfp4(conv)) + }; + func.set_terminator(block, Terminator::Ret(Ret::one(result))); + func +} + +#[test] +fn expanded_nvfp4_dequantize_matches_the_host() { + for block_application in APPLICATIONS { + for global_application in APPLICATIONS { + let mut func = nvfp4_function(true, block_application, global_application); + assert!(expand_nv_fp4(&mut func)); + assert_verifies(&func); + for payload in 0..16u8 { + for scale in [0x00u8, 0x01, 0x08, 0x38, 0x40, 0x7e, 0x7f, 0xb8, 0xff] { + for global in [1.0f32, 0.5, 3.0, -2.0] { + let got = Interp::new(Vec::new()).run( + &func, + &[ + u64::from(payload), + 0, + u64::from(scale), + u64::from(global.to_bits()), + ], + )[0] as u32; + let want = nvfp4::dequantize( + payload, + scale, + global, + block_application, + global_application, + ) + .to_bits(); + assert!( + same_f32(got, want), + "{payload:#x} {scale:#x} {global}: {got:#x} != {want:#x}" + ); + } + } + } } - Opcode::Select(sel) => { - let cond = eval_const(func, sel.cond); - if cond != 0 { - eval_const(func, sel.then) - } else { - eval_const(func, sel.else_) + } +} + +#[test] +fn expanded_nvfp4_quantize_matches_the_host() { + for block_application in APPLICATIONS { + for global_application in APPLICATIONS { + let mut func = nvfp4_function(false, block_application, global_application); + assert!(expand_nv_fp4(&mut func)); + assert_verifies(&func); + for value in sample_floats().into_iter().take(400) { + for scale in [0x38u8, 0x40, 0x7e] { + let got = Interp::new(Vec::new()).run( + &func, + &[ + 0, + u64::from(value.to_bits()), + u64::from(scale), + u64::from(2.0f32.to_bits()), + ], + )[0]; + let want = u64::from(nvfp4::quantize( + value, + scale, + 2.0, + block_application, + global_application, + )); + assert_eq!(got, want, "{value:e} {scale:#x}"); + } } } - Opcode::Convert(cv) => eval_const(func, cv.value), - _ => panic!("unexpected opcode"), + } +} + +#[test] +fn expanded_f32_division_matches_the_hardware() { + let mut func = Function::new(); + let f32t = f32_ty(&mut func); + let block = func.append_block(); + let a = func.append_block_param(block, f32t); + let b = func.append_block_param(block, f32t); + let q = func.append_inst( + block, + f32t, + Opcode::Arith(Arith { + op: BinOp::Div, + lhs: a, + rhs: b, + }), + ); + func.set_terminator(block, Terminator::Ret(Ret::one(q))); + assert!(expand_f32_div(&mut func)); + assert!(!expand_f32_div(&mut func)); + assert_verifies(&func); + for inst in func.block_insts(block).to_vec() { + assert!(!matches!( + func.opcode(inst), + Opcode::Arith(Arith { op: BinOp::Div, .. }) + )); + } + let samples = sample_floats(); + for (i, &x) in samples.iter().enumerate() { + for &y in samples.iter().skip(i % 7).step_by(23).take(60) { + let got = Interp::new(Vec::new()) + .run(&func, &[u64::from(x.to_bits()), u64::from(y.to_bits())])[0] + as u32; + let want = (x / y).to_bits(); + assert!(same_f32(got, want), "{x:e} / {y:e}: {got:#x} != {want:#x}"); + } } } diff --git a/src/ir/src/function.rs b/src/ir/src/function.rs index 23d48d3..456d659 100644 --- a/src/ir/src/function.rs +++ b/src/ir/src/function.rs @@ -3,7 +3,7 @@ extern crate alloc; use alloc::string::String; use alloc::vec::Vec; -use super::attribute::Attribute; +use super::attribute::{Attribute, Endianness}; use super::low_float::Format as LowFloatFormat; use super::nvfp4::ScaleApplication; use super::types::{Type, TypeKind, TypeTable}; @@ -44,22 +44,47 @@ impl Inst { } } +/// Binary operators. +/// +/// Integer types (`iN`) carry no signedness: a value is a bare bit pattern, and the +/// OPERATION decides how it is read. Operations whose result differs between a signed and +/// an unsigned reading therefore come in pairs (`UDiv`/`SDiv`, `Shr`/`Sar`, ...). +/// +/// Operand rules (enforced by `verify`; "int" and "float" include vectors of that lane type): +/// - `Add`, `Sub`, `Mul`: int or float (`Add`/`Sub` also pointer +/- int). +/// - `Div`, `Rem`: FLOAT ONLY (the IEEE division and remainder). +/// - `UDiv`, `SDiv`, `URem`, `SRem`: INT ONLY. Dividing by zero and `MIN / -1` behave exactly +/// as the backends and the emulator implement them. +/// - `BitAnd`, `BitOr`, `BitXor`: int or bool. +/// - `Shl`, `Shr` (logical right shift), `Sar` (arithmetic right shift): INT ONLY. +/// - `UMulh`, `SMulh`: INT ONLY; the high half of the double-width unsigned / signed product. +/// +/// Bitcode-relevant: `as u8` is the wire tag and the values are pinned. #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +#[repr(u8)] pub enum BinOp { - Add, - Sub, - Mul, - Div, - Rem, - BitAnd, - BitOr, - BitXor, - Shl, - Shr, - Mulh, + Add = 0, + Sub = 1, + Mul = 2, + Div = 3, + Rem = 4, + BitAnd = 5, + BitOr = 6, + BitXor = 7, + Shl = 8, + Shr = 9, + Sar = 10, + UDiv = 11, + SDiv = 12, + URem = 13, + SRem = 14, + UMulh = 15, + SMulh = 16, } impl BinOp { + /// The operator spelling of the text syntax (see the `parser` module comment). The + /// signedness suffix is `u` / `s`. pub fn symbol(self) -> &'static str { match self { BinOp::Add => "+", @@ -67,12 +92,18 @@ impl BinOp { BinOp::Mul => "*", BinOp::Div => "/", BinOp::Rem => "%", + BinOp::UDiv => "/u", + BinOp::SDiv => "/s", + BinOp::URem => "%u", + BinOp::SRem => "%s", BinOp::BitAnd => "&", BinOp::BitOr => "|", BinOp::BitXor => "^", BinOp::Shl => "<<", BinOp::Shr => ">>", - BinOp::Mulh => "*h", + BinOp::Sar => ">>s", + BinOp::UMulh => "*hu", + BinOp::SMulh => "*hs", } } } @@ -84,6 +115,9 @@ pub struct Arith { pub rhs: Value, } +/// `op lhs, imm` with the constant `imm` standing in for the right operand. The constant is +/// the one an `Iconst(imm)` of `lhs`'s type would be, so only its low `bits` matter; for a +/// shift it is the shift amount. `op` obeys the same rules as in [`Arith`]. #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct ArithImm { pub op: BinOp, @@ -91,17 +125,37 @@ pub struct ArithImm { pub imm: i64, } +/// Comparison operators; the result is `bool` (for vector operands, a vector of `bool`). +/// +/// Operand rules (enforced by `verify`): both operands have the same type. +/// - `Eq`, `Ne`: any scalar (`bool`, int, float, ptr) or a vector of one. +/// - `Lt`, `Le`, `Gt`, `Ge`: FLOAT ONLY (ordered; false when either operand is NaN). +/// - `Slt`, `Sle`, `Sgt`, `Sge`: INT ONLY, the operands read as two's complement signed. +/// - `Ult`, `Ule`, `Ugt`, `Uge`: int or ptr, the operands read as unsigned. +/// +/// Bitcode-relevant: `as u8` is the wire tag and the values are pinned. #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +#[repr(u8)] pub enum CmpOp { - Eq, - Ne, - Lt, - Le, - Gt, - Ge, + Eq = 0, + Ne = 1, + Lt = 2, + Le = 3, + Gt = 4, + Ge = 5, + Slt = 6, + Sle = 7, + Sgt = 8, + Sge = 9, + Ult = 10, + Ule = 11, + Ugt = 12, + Uge = 13, } impl CmpOp { + /// The operator spelling of the text syntax (see the `parser` module comment). The + /// signedness suffix is `u` / `s`. pub fn symbol(self) -> &'static str { match self { CmpOp::Eq => "==", @@ -110,6 +164,14 @@ impl CmpOp { CmpOp::Le => "<=", CmpOp::Gt => ">", CmpOp::Ge => ">=", + CmpOp::Slt => " "<=s", + CmpOp::Sgt => ">s", + CmpOp::Sge => ">=s", + CmpOp::Ult => " "<=u", + CmpOp::Ugt => ">u", + CmpOp::Uge => ">=u", } } } @@ -139,9 +201,56 @@ pub struct Extract { pub index: u32, } +/// The conversion a [`Convert`] performs. `SiToFp` etc. name the source/destination class and +/// the SIGN of the integer side; widths come from the operand and result types. +/// +/// Type rules (enforced by `verify`; lane counts must match for vectors): +/// - `Trunc`: int -> strictly narrower int. Keeps the low bits. +/// - `Zext`: int or `bool` -> strictly wider int, filling with zeros. +/// - `Sext`: int -> strictly wider int, filling with the sign bit. +/// - `SiToFp` / `UiToFp`: int (read signed / unsigned) -> float. +/// - `FpToSi` / `FpToUi`: float -> int. In-range values truncate toward zero. The result of +/// an out-of-range or NaN input is UNSPECIFIED and differs between backends (aarch64 +/// saturates, x86-64 returns its "integer indefinite" value); frontends that need a +/// specific behavior must range-check or clamp the input first. +/// - `FpResize`: float -> float of a DIFFERENT width (extend or truncate). +/// +/// Same-width int <-> int casts have no kind: the value is simply reused. +/// +/// Bitcode-relevant: encoded as one byte, tag values pinned. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +#[repr(u8)] +pub enum ConvertKind { + Trunc = 0, + Zext = 1, + Sext = 2, + SiToFp = 3, + UiToFp = 4, + FpToSi = 5, + FpToUi = 6, + FpResize = 7, +} + +impl ConvertKind { + /// The mnemonic word of the text syntax: `convert , `. + pub fn name(self) -> &'static str { + match self { + ConvertKind::Trunc => "trunc", + ConvertKind::Zext => "zext", + ConvertKind::Sext => "sext", + ConvertKind::SiToFp => "sitofp", + ConvertKind::UiToFp => "uitofp", + ConvertKind::FpToSi => "fptosi", + ConvertKind::FpToUi => "fptoui", + ConvertKind::FpResize => "fpresize", + } + } +} + #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Convert { pub value: Value, + pub kind: ConvertKind, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -257,17 +366,69 @@ pub struct GlobalAddr { pub via_got: bool, } +/// Memory-access flags shared by [`Load`] and [`Store`]. +/// +/// Bitcode-relevant: see the `MemFlags` encoding in `bitcode.rs`. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] +pub struct MemFlags { + /// The access must not be removed, merged or reordered by optimizers. + pub volatile: bool, + /// Guaranteed alignment of the address in bytes (power of two); 0 = no promise. + pub align: u32, + /// None = ordinary access. Some(o) = atomic load/store with ordering o. + /// Loads allow Relaxed|Acquire|SeqCst; stores allow Relaxed|Release|SeqCst. + pub ordering: Option, + /// Byte order of the memory image. `Native` and `Little` are identical on all current hosts + /// (both backends are little-endian); `Big` requires a byte swap. + pub endian: Endianness, +} + +impl Default for MemFlags { + fn default() -> Self { + Self::new() + } +} + +impl MemFlags { + pub const fn new() -> Self { + Self { + volatile: false, + align: 0, + ordering: None, + endian: Endianness::Native, + } + } + + /// True when the access needs nothing beyond a plain native load/store: + /// not volatile, not atomic, not big-endian. `align` is ignored. + pub fn is_plain(&self) -> bool { + !self.volatile && self.ordering.is_none() && self.endian != Endianness::Big + } +} + #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Load { pub ptr: Value, - pub volatile: bool, + pub mem: MemFlags, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Store { pub value: Value, pub ptr: Value, - pub volatile: bool, + pub mem: MemFlags, +} + +/// Open, target-defined operation identified by an interned name +/// (e.g. `aarch64.aese`). The result, if any, is typed by the instruction. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct Intrinsic { + pub symbol: u32, + pub args: ValueList, + /// One immediate operand (lane index, rounding mode, ...). + pub imm: i64, + /// false = pure (may be removed if the result is unused, or duplicated). + pub side_effects: bool, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -291,11 +452,15 @@ pub struct VaEnd { pub list: Value, } +/// Dot-product accumulate: `acc + sum(a[i] * b[i])` over the lanes. For integer lanes +/// `signed` picks signed (`sdot`) or unsigned (`udot`) multiplication of the lanes; it has no +/// effect on float lanes (the text syntax always spells it, so a parser needs one form). #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct Dot { pub acc: Value, pub a: Value, pub b: Value, + pub signed: bool, } #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -375,18 +540,21 @@ pub struct Barrier { pub scope: BarrierScope, } -/// Bitcode-relevant: names map 1:1 onto NVIDIA encode ops, order pinned. +/// Bitcode-relevant: names map 1:1 onto NVIDIA encode ops, tag values pinned. `SMin`/`SMax` +/// read the operands as signed integers, `UMin`/`UMax` as unsigned. #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] #[repr(u8)] pub enum AtomicOp { Add = 0, - Min = 1, - Max = 2, + SMin = 1, + SMax = 2, BitAnd = 3, BitOr = 4, BitXor = 5, Exchange = 6, CompareExchange = 7, + UMin = 8, + UMax = 9, } /// Bitcode-relevant: encoded as one byte, tag values pinned. @@ -542,6 +710,9 @@ impl<'a> AttrIterator<'a> { #[derive(Clone, PartialEq, Debug)] pub enum Opcode { + /// An integer constant. Integers are signless, so only the low `bits` of the payload + /// matter; `legalize` rewrites the payload of every sub-64-bit constant to the + /// zero-extended form the backends hold values in. Iconst(i64), Fconst(f64), Fconst128(u128), @@ -573,6 +744,7 @@ pub enum Opcode { Matmul(MatMul), Barrier(Barrier), AtomicRmw(AtomicRmw), + Intrinsic(Intrinsic), If(If), } @@ -708,24 +880,6 @@ impl Function { } } - pub fn is_byte_order_tagged(&self, inst: Inst) -> bool { - let mut on_inst = self.attributes_of(AttrTarget::Inst(inst)); - while let Some(attr) = on_inst.next_attr() { - if matches!(attr, Attribute::Endian(_)) { - return true; - } - } - if let Some(result) = self.inst_result(inst) { - let mut on_result = self.attributes_of(AttrTarget::Value(result)); - while let Some(attr) = on_result.next_attr() { - if matches!(attr, Attribute::Endian(_)) { - return true; - } - } - } - false - } - pub fn clone_attrs(&mut self, values: &[ValuePair], insts: &[InstPair]) { let end = self.attributes.len(); for i in 0..end { @@ -868,18 +1022,55 @@ impl Function { } pub fn append_store(&mut self, block: Block, value: Value, ptr: Value) { - self.append_store_vol(block, value, ptr, false); + self.append_store_mem(block, value, ptr, MemFlags::new()); } - pub fn append_store_vol(&mut self, block: Block, value: Value, ptr: Value, volatile: bool) { - self.append_stmt( - block, - Opcode::Store(Store { - value, - ptr, - volatile, - }), - ); + pub fn append_store_mem(&mut self, block: Block, value: Value, ptr: Value, mem: MemFlags) { + self.append_stmt(block, Opcode::Store(Store { value, ptr, mem })); + } + + pub fn append_load(&mut self, block: Block, ty: Type, ptr: Value, mem: MemFlags) -> Value { + self.append_inst(block, ty, Opcode::Load(Load { ptr, mem })) + } + + pub fn append_intrinsic( + &mut self, + block: Block, + ty: Type, + name: &str, + args: &[Value], + imm: i64, + side_effects: bool, + ) -> Value { + let intrinsic = self.make_intrinsic(name, args, imm, side_effects); + self.append_inst(block, ty, Opcode::Intrinsic(intrinsic)) + } + + pub fn append_intrinsic_void( + &mut self, + block: Block, + name: &str, + args: &[Value], + imm: i64, + side_effects: bool, + ) { + let intrinsic = self.make_intrinsic(name, args, imm, side_effects); + self.append_stmt(block, Opcode::Intrinsic(intrinsic)); + } + + fn make_intrinsic( + &mut self, + name: &str, + args: &[Value], + imm: i64, + side_effects: bool, + ) -> Intrinsic { + Intrinsic { + symbol: self.intern_symbol(name), + args: self.intern_values(args), + imm, + side_effects, + } } pub fn append_prefetch(&mut self, block: Block, ptr: Value) { @@ -911,9 +1102,26 @@ impl Function { self.append_stmt(block, Opcode::AtomicRmw(rmw)); } - pub fn append_dot(&mut self, block: Block, acc: Value, a: Value, b: Value) -> Value { + pub fn append_dot( + &mut self, + block: Block, + acc: Value, + a: Value, + b: Value, + signed: bool, + ) -> Value { let ty = self.value_type(acc); - self.append_inst(block, ty, Opcode::Dot(Dot { acc, a, b })) + self.append_inst(block, ty, Opcode::Dot(Dot { acc, a, b, signed })) + } + + pub fn append_convert( + &mut self, + block: Block, + ty: Type, + kind: ConvertKind, + value: Value, + ) -> Value { + self.append_inst(block, ty, Opcode::Convert(Convert { value, kind })) } pub fn append_reduce(&mut self, block: Block, op: BinOp, vector: Value) -> Value { @@ -1346,6 +1554,7 @@ impl Function { match &inst.op { Opcode::StructNew(sn) => lists.push((sn.fields.start, sn.fields.len)), Opcode::Call(c) => lists.push((c.args.start, c.args.len)), + Opcode::Intrinsic(i) => lists.push((i.args.start, i.args.len)), Opcode::CallIndirect(c) => lists.push((c.args.start, c.args.len)), Opcode::If(cf) => { lists.push((cf.then.args.start, cf.then.args.len)); @@ -1416,6 +1625,7 @@ impl Function { } } Opcode::StructNew(_) + | Opcode::Intrinsic(_) | Opcode::Call(_) | Opcode::CallIndirect(_) | Opcode::If(_) => {} @@ -1632,6 +1842,7 @@ impl Function { }), Opcode::Convert(cv) => Opcode::Convert(Convert { value: remap(map, cv.value), + kind: cv.kind, }), Opcode::DecodeLowFloat(cv) => Opcode::DecodeLowFloat(LowFloatConvert { value: remap(map, cv.value), @@ -1676,12 +1887,12 @@ impl Function { } Opcode::Load(ld) => Opcode::Load(Load { ptr: remap(map, ld.ptr), - volatile: ld.volatile, + mem: ld.mem, }), Opcode::Store(st) => Opcode::Store(Store { value: remap(map, st.value), ptr: remap(map, st.ptr), - volatile: st.volatile, + mem: st.mem, }), Opcode::Prefetch(pf) => Opcode::Prefetch(Prefetch { ptr: remap(map, pf.ptr), @@ -1700,6 +1911,7 @@ impl Function { acc: remap(map, d.acc), a: remap(map, d.a), b: remap(map, d.b), + signed: d.signed, }), Opcode::Reduce(red) => Opcode::Reduce(Reduce { vector: remap(map, red.vector), @@ -1722,6 +1934,10 @@ impl Function { } Opcode::AtomicRmw(a) } + Opcode::Intrinsic(mut i) => { + i.args = self.remap_value_list(i.args, map); + Opcode::Intrinsic(i) + } Opcode::If(cond) => Opcode::If(If { cond: remap(map, cond.cond), then: Jump { diff --git a/src/ir/src/legalize.rs b/src/ir/src/legalize.rs index 052bfef..b1cfcf5 100644 --- a/src/ir/src/legalize.rs +++ b/src/ir/src/legalize.rs @@ -2,19 +2,262 @@ extern crate alloc; use alloc::vec::Vec; -use super::function::{BinOp, Block, Function, Opcode, Value}; -use super::types::TypeKind; +use super::attribute::Endianness; +use super::function::{ + Arith, BinOp, Block, Function, Inst, Load, MemFlags, Opcode, Store, Unary, UnaryOp, Value, +}; +use super::types::{IntDesc, Type, TypeKind}; +use super::verify::mem_flags_valid; +/// Lowers the IR to what the native backends select instructions for. +/// +/// Besides struct-param splitting and folding, this removes every big-endian `Load`/`Store` +/// (afterwards `mem.endian` is in {Native, Little}) and every atomic-ordered float access. +/// Atomic-ordered int and pointer accesses stay as `Load`/`Store` with `mem.ordering` set: the +/// backends lower those natively. See [`legalize_memory`] for the rewrites and the +/// little-endian host assumption. +/// +/// Integer constants are signless, so `-1` and `0xffff_ffff` are the same `i32`; this rewrites +/// every sub-64-bit `Iconst` to the canonical zero-extended payload (the form the backends hold +/// values in) and folds constants and immediates width-aware, with the signedness taken from +/// the operation (`UDiv` vs `SDiv`, ...), never from a type. pub fn legalize(func: &mut Function) { split_struct_params(func); let mut subst: Vec<(Value, Value)> = Vec::new(); build_subst(func, &mut subst); apply_subst(func, &subst); + legalize_memory(func); + canonicalize_constants(func); fold_constants(func); fold_immediates(func); drop_dead(func); } +// --- Memory legalization ------------------------------------------------------- + +/// Rewrites big-endian loads/stores and atomic-ordered float accesses into forms the backends +/// select instructions for. +/// +/// ASSUMES A LITTLE-ENDIAN HOST: `Native` and `Little` are treated as identical, and +/// `Big` is realised as a byte swap around a native access. +/// +/// - Atomic int/pointer access -> unchanged: it stays a `Load`/`Store` carrying `mem.ordering` +/// (pointers are 8-byte scalars to the backends, so they pass through like a `u64`). +/// - Atomic float access -> the same-width int access (still atomic) plus +/// `Unary::Reinterpret`, since the backends only order integer-register accesses. +/// - Big endian -> byte swap (shifts, masks, ors in the same int type) after a load / +/// before a store; composed around the (possibly atomic) access when both are requested. +/// Float and pointer accesses go through the same-width int with +/// `Unary::Reinterpret`. +/// +/// `volatile`, `align` and `ordering` stay on the resulting access. Accesses the verifier +/// rejects are left untouched so backends report them. +fn legalize_memory(func: &mut Function) { + for bi in 0..func.block_count() { + let block = Block::from_u32(bi as u32); + let insts = func.block_insts(block).to_vec(); + let mut out: Vec = Vec::with_capacity(insts.len()); + let mut changed = false; + for inst in insts { + match func.opcode(inst) { + Opcode::Load(l) if needs_lowering(&l.mem) => { + changed |= lower_load(func, &mut out, inst, l); + } + Opcode::Store(st) if needs_lowering(&st.mem) => { + changed |= lower_store(func, &mut out, inst, st); + } + _ => {} + } + out.push(inst); + } + if changed { + func.set_block_insts(block, &out); + } + } +} + +fn needs_lowering(mem: &MemFlags) -> bool { + mem.ordering.is_some() || mem.endian == Endianness::Big +} + +/// The integer type an access of `ty` is performed in (`ty` itself when it is an int). +fn access_int_type(func: &mut Function, ty: Type, bytes: u32) -> Type { + match func.types.type_kind(ty) { + TypeKind::Int(_) => ty, + _ => func.types.intern(TypeKind::Int(IntDesc { + bits: (bytes * 8) as u16, + })), + } +} + +fn emit(func: &mut Function, out: &mut Vec, ty: Type, op: Opcode) -> Value { + let value = func.create_inst(ty, op); + out.push(func.defining_inst(value).unwrap()); + value +} + +/// An integer constant of `ty`, wrapped to the type's width and zero-extended (the canonical +/// payload, see [`canonicalize_constants`]). +fn int_const(func: &mut Function, out: &mut Vec, ty: Type, value: u64) -> Value { + let wrapped = match func.types.type_kind(ty) { + TypeKind::Int(d) => zero_extend(value as i64, u32::from(d.bits)), + _ => value as i64, + }; + emit(func, out, ty, Opcode::Iconst(wrapped)) +} + +fn arith( + func: &mut Function, + out: &mut Vec, + ty: Type, + op: BinOp, + lhs: Value, + rhs: Value, +) -> Value { + emit(func, out, ty, Opcode::Arith(Arith { op, lhs, rhs })) +} + +/// Byte `i` of `x` moved to byte position `bytes - 1 - i`. +fn swapped_byte( + func: &mut Function, + out: &mut Vec, + x: Value, + ty: Type, + bytes: u32, + i: u32, +) -> Value { + let j = bytes - 1 - i; + if i < j { + let mask = int_const(func, out, ty, 0xffu64 << (8 * i)); + let low = arith(func, out, ty, BinOp::BitAnd, x, mask); + let amount = int_const(func, out, ty, u64::from(8 * (j - i))); + arith(func, out, ty, BinOp::Shl, low, amount) + } else { + let amount = int_const(func, out, ty, u64::from(8 * (i - j))); + let shifted = arith(func, out, ty, BinOp::Shr, x, amount); + let mask = int_const(func, out, ty, 0xffu64 << (8 * j)); + arith(func, out, ty, BinOp::BitAnd, shifted, mask) + } +} + +/// Emits a byte swap of the `bytes`-wide (2, 4 or 8) int `x` of type `ty` and returns its +/// FINAL operation unemitted, so the caller can place it onto the instruction it replaces. +/// Every term is masked after a right shift, so the (logical) `Shr` is all that is needed. +fn byte_swap(func: &mut Function, out: &mut Vec, x: Value, ty: Type, bytes: u32) -> Opcode { + let mut acc = swapped_byte(func, out, x, ty, bytes, 0); + for i in 1..bytes - 1 { + let term = swapped_byte(func, out, x, ty, bytes, i); + acc = arith(func, out, ty, BinOp::BitOr, acc, term); + } + let term = swapped_byte(func, out, x, ty, bytes, bytes - 1); + Opcode::Arith(Arith { + op: BinOp::BitOr, + lhs: acc, + rhs: term, + }) +} + +fn reinterpret(value: Value) -> Opcode { + Opcode::Unary(Unary { + op: UnaryOp::Reinterpret, + value, + }) +} + +/// `mem` with the byte order lowered away (the byte swap is emitted separately). +fn native_mem(mem: MemFlags) -> MemFlags { + MemFlags { + endian: Endianness::Native, + ..mem + } +} + +fn is_float(func: &Function, ty: Type) -> bool { + matches!(func.types.type_kind(ty), TypeKind::Float(_)) +} + +fn lower_load(func: &mut Function, out: &mut Vec, inst: Inst, l: Load) -> bool { + let Some(result) = func.inst_result(inst) else { + return false; + }; + let ty = func.value_type(result); + if !mem_flags_valid(func, &l.mem, ty, false) { + return false; + } + let Some(bytes) = func.types.scalar_mem_bytes(ty) else { + return false; + }; + let big = l.mem.endian == Endianness::Big && bytes > 1; + let atomic_float = l.mem.ordering.is_some() && is_float(func, ty); + if !big && !atomic_float { + // Nothing to swap or reinterpret: an ordered int/ptr load is native, a one-byte + // big-endian one is just a native load. + if l.mem.endian == Endianness::Big { + *func.opcode_mut(inst) = Opcode::Load(Load { + mem: native_mem(l.mem), + ..l + }); + return true; + } + return false; + } + let int_ty = access_int_type(func, ty, bytes); + + // `pending` always yields the access in `int_ty`; the last step lands on `inst`. + let mut pending = Opcode::Load(Load { + ptr: l.ptr, + mem: native_mem(l.mem), + }); + if big { + let raw = emit(func, out, int_ty, pending); + pending = byte_swap(func, out, raw, int_ty, bytes); + } + if int_ty != ty { + let int_value = emit(func, out, int_ty, pending); + pending = reinterpret(int_value); + } + *func.opcode_mut(inst) = pending; + true +} + +fn lower_store(func: &mut Function, out: &mut Vec, inst: Inst, st: Store) -> bool { + let ty = func.value_type(st.value); + if !mem_flags_valid(func, &st.mem, ty, true) { + return false; + } + let Some(bytes) = func.types.scalar_mem_bytes(ty) else { + return false; + }; + let big = st.mem.endian == Endianness::Big && bytes > 1; + let atomic_float = st.mem.ordering.is_some() && is_float(func, ty); + if !big && !atomic_float { + if st.mem.endian == Endianness::Big { + *func.opcode_mut(inst) = Opcode::Store(Store { + mem: native_mem(st.mem), + ..st + }); + return true; + } + return false; + } + let int_ty = access_int_type(func, ty, bytes); + + let mut value = st.value; + if int_ty != ty { + value = emit(func, out, int_ty, reinterpret(value)); + } + if big { + let swap = byte_swap(func, out, value, int_ty, bytes); + value = emit(func, out, int_ty, swap); + } + *func.opcode_mut(inst) = Opcode::Store(Store { + value, + ptr: st.ptr, + mem: native_mem(st.mem), + }); + true +} + fn split_struct_params(func: &mut Function) { // Split struct-typed block params into scalar field params; rewrite // jump/if edges by extracting fields from the struct argument. @@ -337,6 +580,12 @@ fn apply_subst(func: &mut Function, subst: &[(Value, Value)]) { cf.else_.args = func.intern_values(&oe); Opcode::If(cf) } + Opcode::Intrinsic(mut i) => { + let vals = func.value_list(i.args).to_vec(); + let out: Vec = vals.iter().map(|v| sub(subst, *v)).collect(); + i.args = func.intern_values(&out); + Opcode::Intrinsic(i) + } Opcode::StructNew(sn) => { let vals = func.value_list(sn.fields).to_vec(); let out: Vec = vals.iter().map(|v| sub(subst, *v)).collect(); @@ -381,32 +630,91 @@ fn const_value(func: &Function, v: Value) -> Option { } } -fn fold_arith(op: BinOp, lhs: i64, rhs: i64) -> Option { - match op { - BinOp::Add => Some(lhs.wrapping_add(rhs)), - BinOp::Sub => Some(lhs.wrapping_sub(rhs)), - BinOp::Mul => Some(lhs.wrapping_mul(rhs)), - BinOp::Div => { - if rhs == 0 { - None - } else { - Some(lhs.wrapping_div(rhs)) - } - } - BinOp::Rem => { - if rhs == 0 { - None - } else { - Some(lhs.wrapping_rem(rhs)) +/// `value` wrapped to `bits` and zero-extended: the canonical payload of a `bits`-wide integer. +fn zero_extend(value: i64, bits: u32) -> i64 { + if bits == 0 || bits >= 64 { + value + } else { + (value as u64 & ((1u64 << bits) - 1)) as i64 + } +} + +/// The low `bits` of `value` read as a two's complement signed integer. +fn sign_extend(value: i64, bits: u32) -> i64 { + if bits == 0 || bits >= 64 { + value + } else { + (((value as u64) << (64 - bits)) as i64) >> (64 - bits) + } +} + +/// The width of a constant of `v`'s type: an int's own, a pointer's 64, a bool's 1. +fn value_bits(func: &Function, v: Value) -> Option { + match func.types.type_kind(func.value_type(v)) { + TypeKind::Int(d) => Some(u32::from(d.bits)), + TypeKind::Ptr(_) => Some(64), + TypeKind::Bool => Some(1), + _ => None, + } +} + +/// Integers are signless, so a constant is only its low `bits`: rewrite every sub-64-bit +/// integer `Iconst` to exactly that, zero-extended. The backends hold a value narrower than 64 +/// bits zero-extended and need no further case analysis for a constant. +fn canonicalize_constants(func: &mut Function) { + for i in 0..func.inst_count() { + let inst = super::function::Inst::from_u32(i as u32); + let Opcode::Iconst(c) = func.opcode(inst) else { + continue; + }; + let Some(result) = func.inst_result(inst) else { + continue; + }; + if let TypeKind::Int(d) = func.types.type_kind(func.value_type(result)) { + let canonical = zero_extend(c, u32::from(d.bits)); + if canonical != c { + *func.opcode_mut(inst) = Opcode::Iconst(canonical); } } - BinOp::BitAnd => Some(lhs & rhs), - BinOp::BitOr => Some(lhs | rhs), - BinOp::BitXor => Some(lhs ^ rhs), - BinOp::Shl | BinOp::Shr | BinOp::Mulh => None, } } +/// The value of `lhs op rhs` on `bits`-wide constants, in canonical (zero-extended) form, or +/// `None` when the operation is left to run time: shifts and high multiplies are not folded, +/// and neither is a division by zero or the overflowing `MIN / -1`, so that the backend's own +/// behavior for them is kept. +fn fold_arith(op: BinOp, lhs: i64, rhs: i64, bits: u32) -> Option { + let (ul, ur) = (zero_extend(lhs, bits) as u64, zero_extend(rhs, bits) as u64); + let (sl, sr) = (sign_extend(lhs, bits), sign_extend(rhs, bits)); + let min = if bits == 0 || bits >= 64 { + i64::MIN + } else { + -(1i64 << (bits - 1)) + }; + let folded = match op { + BinOp::Add => lhs.wrapping_add(rhs), + BinOp::Sub => lhs.wrapping_sub(rhs), + BinOp::Mul => lhs.wrapping_mul(rhs), + BinOp::BitAnd => lhs & rhs, + BinOp::BitOr => lhs | rhs, + BinOp::BitXor => lhs ^ rhs, + BinOp::UDiv => ul.checked_div(ur)? as i64, + BinOp::URem => ul.checked_rem(ur)? as i64, + BinOp::SDiv | BinOp::SRem if sl == min && sr == -1 => return None, + BinOp::SDiv => sl.checked_div(sr)?, + BinOp::SRem => sl.checked_rem(sr)?, + // `Div` and `Rem` are floating point; the rest is not folded. + BinOp::Div + | BinOp::Rem + | BinOp::Shl + | BinOp::Shr + | BinOp::Sar + | BinOp::UMulh + | BinOp::SMulh => return None, + }; + Some(zero_extend(folded, bits)) +} + fn fold_constants(func: &mut Function) { for i in 0..func.inst_count() { let inst = super::function::Inst::from_u32(i as u32); @@ -417,16 +725,15 @@ fn fold_constants(func: &mut Function) { let (Some(l), Some(r)) = (const_value(func, lhs), const_value(func, rhs)) else { continue; }; - if let Some(folded) = fold_arith(op, l, r) { + let Some(bits) = value_bits(func, lhs) else { + continue; + }; + if let Some(folded) = fold_arith(op, l, r, bits) { *func.opcode_mut(inst) = Opcode::Iconst(folded); } } } -fn is_unsigned(func: &Function, v: Value) -> bool { - matches!(func.types.type_kind(func.value_type(v)), TypeKind::Int(i) if !i.signed) -} - fn power_of_two_shift(c: i64) -> Option { if c < 2 { return None; @@ -448,6 +755,7 @@ fn imm_has_form(op: BinOp) -> bool { | BinOp::BitXor | BinOp::Shl | BinOp::Shr + | BinOp::Sar ) } @@ -460,12 +768,23 @@ fn imm_commutative(op: BinOp) -> bool { fn imm_fits(op: BinOp, c: i64) -> bool { match op { - BinOp::Shl | BinOp::Shr => (0..=63).contains(&c), + BinOp::Shl | BinOp::Shr | BinOp::Sar => (0..=63).contains(&c), BinOp::Sub => (-2047..=2048).contains(&c), _ => (-2048..=2047).contains(&c), } } +/// The immediate `op` takes for the `bits`-wide constant `c`, if it has an immediate form that +/// fits. Addition and subtraction wrap, so a constant is as well written sign-extended: +/// `x + 0xffff_ffff` on an `i32` is `x + -1`. +fn imm_for(op: BinOp, c: i64, bits: u32) -> Option { + if imm_fits(op, c) { + return Some(c); + } + let signed = sign_extend(c, bits); + (matches!(op, BinOp::Add | BinOp::Sub) && imm_fits(op, signed)).then_some(signed) +} + fn fold_immediates(func: &mut Function) { for i in 0..func.inst_count() { let inst = super::function::Inst::from_u32(i as u32); @@ -493,11 +812,11 @@ fn fold_immediates(func: &mut Function) { } continue; } - if (a.op == BinOp::Div || a.op == BinOp::Rem) && is_unsigned(func, a.lhs) { + if matches!(a.op, BinOp::UDiv | BinOp::URem) { if let Some(c) = const_value(func, a.rhs) && let Some(k) = power_of_two_shift(c) { - if a.op == BinOp::Div { + if a.op == BinOp::UDiv { *func.opcode_mut(inst) = Opcode::ArithImm(super::function::ArithImm { op: BinOp::Shr, lhs: a.lhs, @@ -516,22 +835,23 @@ fn fold_immediates(func: &mut Function) { if !imm_has_form(a.op) { continue; } + let bits = value_bits(func, a.lhs).unwrap_or(64); if let Some(c) = const_value(func, a.rhs) { - if imm_fits(a.op, c) { + if let Some(imm) = imm_for(a.op, c, bits) { *func.opcode_mut(inst) = Opcode::ArithImm(super::function::ArithImm { op: a.op, lhs: a.lhs, - imm: c, + imm, }); } } else if imm_commutative(a.op) && let Some(c) = const_value(func, a.lhs) - && imm_fits(a.op, c) + && let Some(imm) = imm_for(a.op, c, bits) { *func.opcode_mut(inst) = Opcode::ArithImm(super::function::ArithImm { op: a.op, lhs: a.rhs, - imm: c, + imm, }); } } @@ -560,7 +880,7 @@ fn is_pure(op: &Opcode) -> bool { | Opcode::Dot(_) | Opcode::Reduce(_) | Opcode::Splat(_) - ) + ) || matches!(op, Opcode::Intrinsic(i) if !i.side_effects) } fn count_uses(func: &Function, uses: &mut [u32]) { @@ -658,6 +978,11 @@ fn count_uses(func: &Function, uses: &mut [u32]) { use_one(v); } } + Opcode::Intrinsic(i) => { + for v in func.value_list(i.args).to_vec() { + use_one(v); + } + } Opcode::Iconst(_) | Opcode::Fconst(_) | Opcode::Fconst128(_) @@ -711,3 +1036,692 @@ fn drop_dead(func: &mut Function) { } } } + +#[cfg(test)] +mod tests { + use alloc::vec; + use alloc::vec::Vec; + use std::collections::HashMap; + + use super::*; + use crate::function::{AtomicOrdering, Ret, Terminator}; + use crate::types::{AddressSpace, FloatKind}; + use crate::verify::{Profile, verify}; + + /// Straight-line interpreter over one block, enough to execute legalized memory + /// accesses. Values are raw bits masked to their type's width; the address of the + /// pointer parameter is 0. Asserts it never sees an access `legalize` must remove. + struct Machine { + mem: Vec, + values: HashMap, + } + + fn bits_of(func: &Function, v: Value) -> u32 { + match func.types.type_kind(func.value_type(v)) { + TypeKind::Int(d) => u32::from(d.bits), + TypeKind::Float(FloatKind::F16) => 16, + TypeKind::Float(FloatKind::F32) => 32, + TypeKind::Float(FloatKind::F64) => 64, + TypeKind::Ptr(_) => 64, + other => panic!("unexpected type {other:?}"), + } + } + + fn mask(bits: u32) -> u64 { + if bits >= 64 { !0 } else { (1u64 << bits) - 1 } + } + + impl Machine { + fn new(mem: &[u8]) -> Self { + Self { + mem: mem.to_vec(), + values: HashMap::new(), + } + } + + fn read(&self, addr: u64, bytes: u32) -> u64 { + let a = addr as usize; + (0..bytes as usize).fold(0u64, |acc, i| acc | u64::from(self.mem[a + i]) << (8 * i)) + } + + fn write(&mut self, addr: u64, bytes: u32, value: u64) { + for i in 0..bytes as usize { + self.mem[addr as usize + i] = (value >> (8 * i)) as u8; + } + } + + fn binary(func: &Function, lhs: Value, op: BinOp, l: u64, r: u64) -> u64 { + let bits = bits_of(func, lhs); + let raw = match op { + BinOp::BitAnd => l & r, + BinOp::BitOr => l | r, + BinOp::Shl => l << r, + BinOp::Shr => l >> r, + other => panic!("unexpected op {other:?}"), + }; + raw & mask(bits) + } + + fn run(&mut self, func: &Function, block: Block, args: &[(Value, u64)]) { + self.values.extend(args.iter().copied()); + for &inst in func.block_insts(block) { + let result = func.inst_result(inst); + let out = match func.opcode(inst) { + Opcode::Iconst(c) => Some((c as u64) & mask(bits_of(func, result.unwrap()))), + Opcode::Arith(a) => { + let (l, r) = (self.values[&a.lhs], self.values[&a.rhs]); + Some(Self::binary(func, a.lhs, a.op, l, r)) + } + Opcode::ArithImm(a) => { + let l = self.values[&a.lhs]; + Some(Self::binary(func, a.lhs, a.op, l, a.imm as u64)) + } + Opcode::Unary(u) => { + assert_eq!(u.op, UnaryOp::Reinterpret); + let bits = bits_of(func, result.unwrap()); + assert_eq!(bits, bits_of(func, u.value)); + Some(self.values[&u.value] & mask(bits)) + } + // On a single thread an ordered access is an ordinary one. + Opcode::Load(l) => { + assert!(l.mem.endian != Endianness::Big); + let bytes = bits_of(func, result.unwrap()) / 8; + Some(self.read(self.values[&l.ptr], bytes)) + } + Opcode::Store(st) => { + assert!(st.mem.endian != Endianness::Big); + let bytes = bits_of(func, st.value) / 8; + self.write(self.values[&st.ptr], bytes, self.values[&st.value]); + None + } + other => panic!("unexpected opcode {other:?}"), + }; + if let (Some(r), Some(v)) = (result, out) { + self.values.insert(r, v); + } + } + } + } + + fn int_ty(func: &mut Function, bits: u16) -> Type { + func.types.intern(TypeKind::Int(IntDesc { bits })) + } + + fn mem(ordering: Option, endian: Endianness) -> MemFlags { + MemFlags { + ordering, + endian, + ..MemFlags::new() + } + } + + /// `fn(p: ptr) -> ty { ret load. p }`; returns the function, entry block and loaded value. + fn load_fn(ty: TypeKind, m: MemFlags) -> (Function, Block, Value, Value) { + let mut func = Function::new(); + let ty = func.types.intern(ty); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let v = func.append_load(entry, ty, p, m); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + (func, entry, p, v) + } + + /// `fn(p: ptr, x: ty) { store. x, p }`. + fn store_fn(ty: TypeKind, m: MemFlags) -> (Function, Block, Value, Value) { + let mut func = Function::new(); + let ty = func.types.intern(ty); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let x = func.append_block_param(entry, ty); + func.append_store_mem(entry, x, p, m); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + (func, entry, p, x) + } + + fn assert_legal(func: &Function) { + let diags = verify(func, Profile::High); + assert!(diags.ok(), "{:?}", diags.items()); + for bi in 0..func.block_count() { + for &inst in func.block_insts(Block::from_u32(bi as u32)) { + match func.opcode(inst) { + Opcode::Load(l) => { + assert!(l.mem.endian != Endianness::Big); + if l.mem.ordering.is_some() { + assert!(!is_float( + func, + func.value_type(func.inst_result(inst).unwrap()) + )); + } + } + Opcode::Store(st) => { + assert!(st.mem.endian != Endianness::Big); + if st.mem.ordering.is_some() { + assert!(!is_float(func, func.value_type(st.value))); + } + } + _ => {} + } + } + } + } + + fn ops(func: &Function) -> Vec { + func.block_insts(Block::from_u32(0)) + .iter() + .map(|&i| func.opcode(i)) + .collect() + } + + fn int(bits: u16) -> TypeKind { + TypeKind::Int(IntDesc { bits }) + } + + #[test] + fn big_endian_loads_become_a_native_load_and_a_byte_swap() { + let image: [u8; 8] = [0xf1, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + for kind in [int(16), int(32), int(64)] { + let bytes = match &kind { + TypeKind::Int(d) => usize::from(d.bits / 8), + _ => unreachable!(), + }; + let (mut func, entry, p, v) = load_fn(kind.clone(), mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + let loads = ops(&func) + .into_iter() + .filter(|o| matches!(o, Opcode::Load(_))) + .count(); + assert_eq!(loads, 1, "{kind:?}"); + let mut m = Machine::new(&image); + m.run(&func, entry, &[(p, 0)]); + let want = image[..bytes] + .iter() + .fold(0u64, |acc, &b| acc << 8 | u64::from(b)); + assert_eq!(m.values[&v], want, "{kind:?}"); + } + } + + #[test] + fn big_endian_float_loads_swap_in_an_int_and_reinterpret() { + let (mut func, entry, p, v) = + load_fn(TypeKind::Float(FloatKind::F32), mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + assert!(matches!( + ops(&func).last(), + Some(Opcode::Unary(u)) if u.op == UnaryOp::Reinterpret + )); + let mut m = Machine::new(&1.5f32.to_bits().to_be_bytes()); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], u64::from(1.5f32.to_bits())); + } + + #[test] + fn big_endian_stores_swap_before_a_native_store() { + for (kind, value, want) in [ + (int(16), 0x0102u64, vec![0x01u8, 0x02]), + (int(32), 0x0102_0304, vec![1, 2, 3, 4]), + (int(64), 0x0102_0304_0506_0708, vec![1, 2, 3, 4, 5, 6, 7, 8]), + ] { + let (mut func, entry, p, x) = store_fn(kind, mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + let mut m = Machine::new(&[0xee; 8]); + m.run(&func, entry, &[(p, 0), (x, value)]); + assert_eq!(&m.mem[..want.len()], &want[..]); + assert!(m.mem[want.len()..].iter().all(|&b| b == 0xee)); + } + let (mut func, entry, p, x) = + store_fn(TypeKind::Float(FloatKind::F32), mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + let mut m = Machine::new(&[0; 4]); + m.run(&func, entry, &[(p, 0), (x, u64::from(2.5f32.to_bits()))]); + assert_eq!(m.mem, 2.5f32.to_bits().to_be_bytes()); + } + + #[test] + fn volatile_and_align_stay_on_the_plain_access() { + let flags = MemFlags { + volatile: true, + align: 4, + ordering: None, + endian: Endianness::Big, + }; + let (mut func, ..) = load_fn(int(32), flags); + legalize(&mut func); + let plain: Vec = ops(&func) + .into_iter() + .filter_map(|o| match o { + Opcode::Load(l) => Some(l.mem), + _ => None, + }) + .collect(); + assert_eq!( + plain, + [MemFlags { + volatile: true, + align: 4, + ordering: None, + endian: Endianness::Native, + }] + ); + let (mut func, ..) = store_fn(int(32), flags); + legalize(&mut func); + assert!(ops(&func).iter().any(|o| matches!( + o, + Opcode::Store(st) if st.mem.volatile && st.mem.align == 4 + && st.mem.endian == Endianness::Native + ))); + } + + #[test] + fn one_byte_big_endian_needs_no_swap() { + let (mut func, entry, p, v) = load_fn(int(8), mem(None, Endianness::Big)); + legalize(&mut func); + assert_legal(&func); + assert_eq!(ops(&func).len(), 1); + let mut m = Machine::new(&[0x7f]); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], 0x7f); + } + + #[test] + fn native_and_little_accesses_are_left_alone() { + for endian in [Endianness::Native, Endianness::Little] { + let flags = MemFlags { + volatile: true, + align: 8, + endian, + ..MemFlags::new() + }; + let (mut func, ..) = load_fn(int(64), flags); + let before = ops(&func); + legalize(&mut func); + assert_eq!(ops(&func), before); + } + } + + #[test] + fn atomic_int_loads_stay_ordered_loads() { + for ordering in [ + AtomicOrdering::Relaxed, + AtomicOrdering::Acquire, + AtomicOrdering::SeqCst, + ] { + for kind in [int(8), int(16), int(32), int(64)] { + let flags = mem(Some(ordering), Endianness::Native); + let (mut func, entry, p, v) = load_fn(kind.clone(), flags); + let before = ops(&func); + legalize(&mut func); + assert_legal(&func); + assert_eq!(ops(&func), before, "{kind:?} {ordering:?}"); + assert!(matches!( + ops(&func)[..], + [Opcode::Load(l)] if l.mem.ordering == Some(ordering) + )); + let mut m = Machine::new(&[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]); + m.run(&func, entry, &[(p, 0)]); + let bytes = match &kind { + TypeKind::Int(d) => u32::from(d.bits / 8), + _ => unreachable!(), + }; + assert_eq!(m.values[&v], m.read(0, bytes), "{kind:?}"); + assert_eq!( + &m.mem[..], + &[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88] + ); + } + } + } + + #[test] + fn atomic_float_loads_go_through_an_atomic_int_load() { + let seq = Some(AtomicOrdering::SeqCst); + let (mut func, entry, p, v) = load_fn( + TypeKind::Float(FloatKind::F64), + mem(seq, Endianness::Native), + ); + legalize(&mut func); + assert_legal(&func); + let ops = ops(&func); + assert_eq!(ops.len(), 2); + assert!(matches!( + ops[0], + Opcode::Load(l) if l.mem.ordering == seq + )); + assert!(matches!(ops[1], Opcode::Unary(u) if u.op == UnaryOp::Reinterpret)); + let mut m = Machine::new(&(-4.25f64).to_bits().to_le_bytes()); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], (-4.25f64).to_bits()); + } + + #[test] + fn atomic_pointer_accesses_pass_through_unchanged() { + let seq = Some(AtomicOrdering::SeqCst); + let (mut func, entry, p, v) = load_fn( + TypeKind::Ptr(AddressSpace::Global), + mem(seq, Endianness::Native), + ); + let before = ops(&func); + legalize(&mut func); + assert_legal(&func); + assert_eq!(ops(&func), before); + let mut m = Machine::new(&0x0123_4567_89ab_cdefu64.to_le_bytes()); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], 0x0123_4567_89ab_cdef); + + let (mut func, ..) = store_fn( + TypeKind::Ptr(AddressSpace::Global), + mem(Some(AtomicOrdering::Release), Endianness::Native), + ); + let before = ops(&func); + legalize(&mut func); + assert_eq!(ops(&func), before); + } + + #[test] + fn atomic_int_stores_stay_ordered_stores() { + for ordering in [ + AtomicOrdering::Relaxed, + AtomicOrdering::Release, + AtomicOrdering::SeqCst, + ] { + let (mut func, entry, p, x) = + store_fn(int(32), mem(Some(ordering), Endianness::Native)); + let before = ops(&func); + legalize(&mut func); + assert_legal(&func); + assert_eq!(ops(&func), before); + assert!(matches!( + ops(&func)[..], + [Opcode::Store(s)] if s.mem.ordering == Some(ordering) && s.ptr == p && s.value == x + )); + let mut m = Machine::new(&[0; 4]); + m.run(&func, entry, &[(p, 0), (x, 0xdead_beef)]); + assert_eq!(m.mem, 0xdead_beefu32.to_le_bytes()); + } + } + + #[test] + fn atomic_float_stores_reinterpret_into_an_atomic_int_store() { + let release = Some(AtomicOrdering::Release); + let (mut func, entry, p, x) = store_fn( + TypeKind::Float(FloatKind::F32), + mem(release, Endianness::Native), + ); + legalize(&mut func); + assert_legal(&func); + let ops = ops(&func); + assert_eq!(ops.len(), 2); + assert!(matches!(ops[0], Opcode::Unary(u) if u.op == UnaryOp::Reinterpret)); + assert!(matches!( + ops[1], + Opcode::Store(s) if s.mem.ordering == release + )); + let mut m = Machine::new(&[0; 4]); + m.run(&func, entry, &[(p, 0), (x, u64::from(0.5f32.to_bits()))]); + assert_eq!(m.mem, 0.5f32.to_bits().to_le_bytes()); + } + + #[test] + fn atomic_and_big_endian_compose() { + let flags = mem(Some(AtomicOrdering::Acquire), Endianness::Big); + let (mut func, entry, p, v) = load_fn(int(32), flags); + legalize(&mut func); + assert_legal(&func); + let mut m = Machine::new(&[0xde, 0xad, 0xbe, 0xef]); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], 0xdead_beef); + + let flags = mem(Some(AtomicOrdering::Release), Endianness::Big); + let (mut func, entry, p, x) = store_fn(int(32), flags); + legalize(&mut func); + assert_legal(&func); + assert!(ops(&func).iter().any(|o| matches!( + o, + Opcode::Store(s) if s.mem.ordering == Some(AtomicOrdering::Release) + && s.mem.endian == Endianness::Native + ))); + let mut m = Machine::new(&[0; 4]); + m.run(&func, entry, &[(p, 0), (x, 0xdead_beef)]); + assert_eq!(m.mem, [0xde, 0xad, 0xbe, 0xef]); + + let flags = mem(Some(AtomicOrdering::SeqCst), Endianness::Big); + let (mut func, entry, p, v) = load_fn(TypeKind::Float(FloatKind::F32), flags); + legalize(&mut func); + assert_legal(&func); + let mut m = Machine::new(&1.25f32.to_bits().to_be_bytes()); + m.run(&func, entry, &[(p, 0)]); + assert_eq!(m.values[&v], u64::from(1.25f32.to_bits())); + } + + #[test] + fn every_valid_flag_combination_legalizes_to_verifiable_plain_ops() { + use AtomicOrdering::*; + let mut func = Function::new(); + let i32_t = int_ty(&mut func, 32); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let mut last = None; + for volatile in [false, true] { + for align in [0, 4] { + for endian in [Endianness::Native, Endianness::Little, Endianness::Big] { + for (load_o, store_o) in [ + (None, None), + (Some(Relaxed), Some(Relaxed)), + (Some(Acquire), Some(Release)), + (Some(SeqCst), Some(SeqCst)), + ] { + let l = MemFlags { + volatile, + align, + ordering: load_o, + endian, + }; + let s = MemFlags { + volatile, + align, + ordering: store_o, + endian, + }; + let v = func.append_load(entry, i32_t, p, l); + func.append_store_mem(entry, v, p, s); + last = Some(v); + } + } + } + } + func.set_terminator(entry, Terminator::Ret(Ret::one(last.unwrap()))); + assert!(verify(&func, Profile::High).ok()); + legalize(&mut func); + assert_legal(&func); + } + + #[test] + fn accesses_the_verifier_rejects_are_left_untouched() { + // A release load: there is no sound lowering, so a backend must see it and refuse. + let (mut func, ..) = load_fn( + int(32), + mem(Some(AtomicOrdering::Release), Endianness::Native), + ); + let before = ops(&func); + legalize(&mut func); + assert_eq!(ops(&func), before); + // A big-endian aggregate. + let (mut func, ..) = load_fn(TypeKind::Struct(vec![]), mem(None, Endianness::Big)); + let before = ops(&func); + legalize(&mut func); + assert_eq!(ops(&func), before); + } + + #[test] + fn pure_unused_intrinsics_are_dropped_and_effectful_ones_kept() { + let mut func = Function::new(); + let i32_t = int_ty(&mut func, 32); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let _dead = func.append_intrinsic(entry, i32_t, "t.pure", &[a], 0, false); + let _kept = func.append_intrinsic(entry, i32_t, "t.effect", &[a], 0, true); + func.set_terminator(entry, Terminator::Ret(Ret::one(a))); + legalize(&mut func); + let names: Vec<&str> = ops(&func) + .iter() + .map(|o| match o { + Opcode::Intrinsic(i) => func.symbol_name(i.symbol), + _ => panic!("unexpected op"), + }) + .collect(); + assert_eq!(names, ["t.effect"]); + } + + /// The defining op of the value `fn() { ret }` returns after `legalize`, over + /// constants of `bits` width. + fn folded(bits: u16, op: BinOp, lhs: i64, rhs: i64) -> Opcode { + let mut func = Function::new(); + let ty = int_ty(&mut func, bits); + let entry = func.append_block(); + let a = func.append_inst(entry, ty, Opcode::Iconst(lhs)); + let b = func.append_inst(entry, ty, Opcode::Iconst(rhs)); + let v = func.append_inst(entry, ty, Opcode::Arith(Arith { op, lhs: a, rhs: b })); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + legalize(&mut func); + func.opcode(func.defining_inst(v).unwrap()) + } + + /// The defining op of the value `fn(x: i32) { ret x c }` returns after `legalize`. + fn with_constant(op: BinOp, c: i64) -> Opcode { + let mut func = Function::new(); + let ty = int_ty(&mut func, 32); + let entry = func.append_block(); + let x = func.append_block_param(entry, ty); + let k = func.append_inst(entry, ty, Opcode::Iconst(c)); + let v = func.append_inst(entry, ty, Opcode::Arith(Arith { op, lhs: x, rhs: k })); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + legalize(&mut func); + func.opcode(func.defining_inst(v).unwrap()) + } + + #[test] + fn integer_constants_become_their_zero_extended_bits() { + let mut func = Function::new(); + let (i8_t, i32_t, i64_t) = ( + int_ty(&mut func, 8), + int_ty(&mut func, 32), + int_ty(&mut func, 64), + ); + let entry = func.append_block(); + let minus_one = func.append_inst(entry, i32_t, Opcode::Iconst(-1)); + let wide = func.append_inst(entry, i8_t, Opcode::Iconst(0x1_0000_00ff)); + let all_ones = func.append_inst(entry, i64_t, Opcode::Iconst(-1)); + let min = func.append_inst(entry, i8_t, Opcode::Iconst(-128)); + func.set_terminator( + entry, + Terminator::Ret(Ret::many(&[minus_one, wide, all_ones, min])), + ); + legalize(&mut func); + let payload = |v: Value| match func.opcode(func.defining_inst(v).unwrap()) { + Opcode::Iconst(c) => c, + other => panic!("expected a constant, got {other:?}"), + }; + assert_eq!(payload(minus_one), 0xffff_ffff); + assert_eq!(payload(wide), 0xff); + assert_eq!(payload(all_ones), -1); + assert_eq!(payload(min), 0x80); + } + + #[test] + fn folding_reads_the_operands_as_the_operation_says() { + use BinOp::*; + // (bits, operator, lhs, rhs, the canonical zero-extended result) + let cases: &[(u16, BinOp, i64, i64, i64)] = &[ + (32, UDiv, -1, 2, 0x7fff_ffff), + (32, SDiv, -1, 2, 0), + (32, SDiv, -6, 2, 0xffff_fffd), + (32, URem, -1, 10, 5), + (32, SRem, -7, 3, 0xffff_ffff), + (8, UDiv, 200, 3, 66), + (8, SDiv, 200, 3, 0xee), + (8, SRem, 200, 3, 0xfe), + (8, Add, 200, 100, 44), + (8, Sub, 1, 2, 0xff), + (8, Mul, 16, 16, 0), + (16, BitXor, 0xffff, 0x00ff, 0xff00), + (64, UDiv, -1, 2, i64::MAX), + (64, SDiv, -1, 2, 0), + (64, URem, -1, 10, 5), + ]; + for &(bits, op, lhs, rhs, want) in cases { + match folded(bits, op, lhs, rhs) { + Opcode::Iconst(got) => { + assert_eq!(got, want, "i{bits} {op:?} {lhs} {rhs}"); + } + other => panic!("i{bits} {op:?} {lhs} {rhs} did not fold: {other:?}"), + } + } + } + + #[test] + fn folding_leaves_what_the_backend_defines_to_run_time() { + use BinOp::*; + let cases: &[(u16, BinOp, i64, i64)] = &[ + (32, UDiv, 5, 0), + (32, URem, 5, 0), + (32, SDiv, 5, 0), + (32, SRem, 5, 0), + (32, SDiv, i64::from(i32::MIN), -1), + (32, SRem, i64::from(i32::MIN), -1), + (8, SDiv, 0x80, 0xff), + (64, SDiv, i64::MIN, -1), + (64, SRem, i64::MIN, -1), + (32, UMulh, 3, 5), + (32, SMulh, 3, 5), + ]; + for &(bits, op, lhs, rhs) in cases { + assert!( + matches!(folded(bits, op, lhs, rhs), Opcode::Arith(a) if a.op == op), + "i{bits} {op:?} {lhs} {rhs}" + ); + } + } + + #[test] + fn immediates_follow_the_operation() { + use BinOp::*; + let imm = |op, c| match with_constant(op, c) { + Opcode::ArithImm(a) => { + assert_eq!(a.lhs, Value::from_u32(0), "{op:?} {c}"); + Some((a.op, a.imm)) + } + Opcode::Arith(a) => { + assert_eq!(a.op, op, "{op:?} {c}"); + None + } + other => panic!("unexpected {other:?}"), + }; + // Unsigned division and remainder by a power of two are a shift and a mask ... + assert_eq!(imm(UDiv, 8), Some((Shr, 3))); + assert_eq!(imm(URem, 8), Some((BitAnd, 7))); + assert_eq!(imm(URem, 4096), None); + // ... the signed forms and the high multiplies are not. + assert_eq!(imm(SDiv, 8), None); + assert_eq!(imm(SRem, 8), None); + assert_eq!(imm(UMulh, 2), None); + assert_eq!(imm(SMulh, 2), None); + // Both right shifts and the left shift have an immediate form. + assert_eq!(imm(Shr, 3), Some((Shr, 3))); + assert_eq!(imm(Sar, 3), Some((Sar, 3))); + assert_eq!(imm(Shl, 3), Some((Shl, 3))); + assert_eq!(imm(Sar, 64), None); + assert_eq!(imm(Mul, 16), Some((Shl, 4))); + // A constant that is -1 read signed is still `-1` for add / sub, which wrap. + assert_eq!(imm(Add, -1), Some((Add, -1))); + assert_eq!(imm(Sub, -1), Some((Sub, -1))); + assert_eq!(imm(Add, 0x7ff), Some((Add, 0x7ff))); + assert_eq!(imm(Add, 0x800), None); + assert_eq!(imm(BitAnd, 0xff), Some((BitAnd, 0xff))); + } +} diff --git a/src/ir/src/lib.rs b/src/ir/src/lib.rs index 9175cfe..c31f48b 100755 --- a/src/ir/src/lib.rs +++ b/src/ir/src/lib.rs @@ -1,17 +1,19 @@ -// NOTE: volt-ir requires std: pliron (and its dependencies) need std to -// build. no_std support is deferred until pliron can build without std. +#![no_std] extern crate alloc; +#[cfg(test)] +extern crate std; + pub mod attribute; pub mod bitcode; pub mod builder; pub mod critical_edge; -pub mod dialect; pub mod entity; pub mod expand; pub mod function; pub mod legalize; pub mod low_float; +pub mod module; pub mod nvfp4; pub mod parser; pub mod reachable; diff --git a/src/ir/src/module.rs b/src/ir/src/module.rs new file mode 100644 index 0000000..dca8063 --- /dev/null +++ b/src/ir/src/module.rs @@ -0,0 +1,505 @@ +//! A translation unit: named functions plus named data globals. +//! +//! [`Module`] is the single description of a unit of code and data shared by +//! every consumer (object writers, in-process linkers, JITs). Symbols that are +//! referenced but not defined here (external functions or data) are not an +//! error at this level; the linker or host resolves them. + +use alloc::string::{String, ToString as _}; +use alloc::vec::Vec; +use core::fmt; + +use crate::function::Function; +use crate::verify::{self, Diagnostic, Profile}; + +/// Size in bytes of a data relocation slot: a pointer-sized absolute address. +pub const RELOC_SLOT_SIZE: u64 = 8; + +/// The section a data global lands in: read-only, writable, or zero-initialized. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum DataKind { + Rodata, + Data, + Bss, +} + +/// An internal relocation within a global. At byte `offset` within the global, +/// the linker writes the pointer-sized absolute runtime address of `symbol`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct DataReloc { + pub offset: usize, + pub symbol: String, +} + +/// A named data global. For [`DataKind::Bss`], `bytes` is empty and `size` +/// gives the zero-initialized length; otherwise `size == bytes.len()`. +/// +/// `align` is the required alignment in bytes: `0` selects the natural +/// alignment ([`natural_align`] of `size`), otherwise it must be a power of two. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Global { + pub name: String, + pub kind: DataKind, + pub bytes: Vec, + pub size: u64, + pub align: u32, + pub relocs: Vec, +} + +impl Global { + /// The alignment this global is laid out with: `align`, or the natural + /// alignment of `size` when `align` is `0`. + pub fn effective_align(&self) -> usize { + match self.align { + 0 => natural_align(self.size), + a => a as usize, + } + } +} + +/// A data object's natural alignment: the next power of two up to `size`, +/// capped at 8. Covers scalars and pointers up to a 64-bit word. +pub fn natural_align(size: u64) -> usize { + size.max(1).checked_next_power_of_two().unwrap_or(8).min(8) as usize +} + +/// A named function definition. +#[derive(Clone, Debug)] +pub struct FunctionDef { + pub name: String, + pub function: Function, +} + +/// Why [`Module::verify`] rejected a module. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ModuleError { + /// Two definitions (function or global) share a name. + DuplicateName(String), + /// A non-bss global's `size` differs from its `bytes.len()`. + SizeMismatch { name: String, size: u64, len: usize }, + /// A bss global carries bytes or relocations. + BssInitialized(String), + /// A relocation slot does not fit within its global. + RelocOutOfRange { name: String, offset: usize }, + /// A global's `align` is neither `0` nor a power of two. + InvalidAlign { name: String, align: u32 }, + /// A function failed IR verification. + InvalidFunction { + name: String, + diagnostics: Vec, + }, +} + +impl fmt::Display for ModuleError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DuplicateName(name) => write!(f, "duplicate module symbol {name}"), + Self::SizeMismatch { name, size, len } => { + write!( + f, + "global {name} has size {size} but {len} initializer bytes" + ) + } + Self::BssInitialized(name) => { + write!(f, "bss global {name} has initializer bytes or relocations") + } + Self::RelocOutOfRange { name, offset } => { + write!( + f, + "relocation at offset {offset} does not fit in global {name}" + ) + } + Self::InvalidAlign { name, align } => { + write!(f, "global {name} has invalid alignment {align}") + } + Self::InvalidFunction { name, diagnostics } => { + write!(f, "function {name} failed verification")?; + if let Some(first) = diagnostics.first() { + write!(f, ": {first:?}")?; + if diagnostics.len() > 1 { + write!(f, " (and {} more)", diagnostics.len() - 1)?; + } + } + Ok(()) + } + } + } +} + +impl core::error::Error for ModuleError {} + +/// A set of named functions and data globals. Function order is preserved; +/// linkers place the first function at the start of the code image. +#[derive(Clone, Debug, Default)] +pub struct Module { + pub functions: Vec, + pub globals: Vec, +} + +impl Module { + pub fn new() -> Module { + Module::default() + } + + pub fn add_function(&mut self, name: &str, function: Function) { + self.functions.push(FunctionDef { + name: name.to_string(), + function, + }); + } + + /// Add a named read-only data blob (a global constant). + pub fn add_data(&mut self, name: &str, bytes: Vec) { + self.add_data_relocs(name, bytes, Vec::new()); + } + + /// Add a named read-only data blob carrying internal relocations. + pub fn add_data_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { + self.add_initialized(name, DataKind::Rodata, bytes, relocs); + } + + /// Add a named writable data global. + pub fn add_writable(&mut self, name: &str, bytes: Vec) { + self.add_writable_relocs(name, bytes, Vec::new()); + } + + /// Add a named writable data global carrying internal relocations. + pub fn add_writable_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { + self.add_initialized(name, DataKind::Data, bytes, relocs); + } + + /// Add a named zero-initialized data global of `size` bytes. + pub fn add_bss(&mut self, name: &str, size: u64) { + self.globals.push(Global { + name: name.to_string(), + kind: DataKind::Bss, + bytes: Vec::new(), + size, + align: 0, + relocs: Vec::new(), + }); + } + + fn add_initialized( + &mut self, + name: &str, + kind: DataKind, + bytes: Vec, + relocs: Vec, + ) { + self.globals.push(Global { + name: name.to_string(), + kind, + size: bytes.len() as u64, + bytes, + align: 0, + relocs, + }); + } + + /// The function defined under `name`. + pub fn function(&self, name: &str) -> Option<&Function> { + self.functions + .iter() + .find(|f| f.name == name) + .map(|f| &f.function) + } + + /// The global defined under `name`. + pub fn global(&self, name: &str) -> Option<&Global> { + self.globals.iter().find(|g| g.name == name) + } + + /// Check module-level well-formedness and verify every function. + /// + /// Undefined symbols (calls, `global_addr`, data relocations naming + /// something not defined here) are allowed: they are external. + pub fn verify(&self) -> Result<(), ModuleError> { + let mut names: Vec<&str> = self + .functions + .iter() + .map(|f| f.name.as_str()) + .chain(self.globals.iter().map(|g| g.name.as_str())) + .collect(); + names.sort_unstable(); + if let Some(pair) = names.windows(2).find(|pair| pair[0] == pair[1]) { + return Err(ModuleError::DuplicateName(pair[0].to_string())); + } + for g in &self.globals { + self.verify_global(g)?; + } + for f in &self.functions { + let diagnostics = verify::verify(&f.function, Profile::High); + if !diagnostics.ok() { + return Err(ModuleError::InvalidFunction { + name: f.name.clone(), + diagnostics: diagnostics.items().to_vec(), + }); + } + } + Ok(()) + } + + fn verify_global(&self, g: &Global) -> Result<(), ModuleError> { + if g.align != 0 && !g.align.is_power_of_two() { + return Err(ModuleError::InvalidAlign { + name: g.name.clone(), + align: g.align, + }); + } + if g.kind == DataKind::Bss { + if !g.bytes.is_empty() || !g.relocs.is_empty() { + return Err(ModuleError::BssInitialized(g.name.clone())); + } + return Ok(()); + } + if g.size != g.bytes.len() as u64 { + return Err(ModuleError::SizeMismatch { + name: g.name.clone(), + size: g.size, + len: g.bytes.len(), + }); + } + for r in &g.relocs { + let fits = (r.offset as u64) + .checked_add(RELOC_SLOT_SIZE) + .is_some_and(|end| end <= g.size); + if !fits { + return Err(ModuleError::RelocOutOfRange { + name: g.name.clone(), + offset: r.offset, + }); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + use crate::function::{MemFlags, Ret, Terminator}; + + fn constant(value: i64) -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let v = f.append_inst(block, ty, crate::function::Opcode::Iconst(value)); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + f + } + + /// A function the verifier rejects: a load with a non-power-of-two `align`. + fn malformed() -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i32").unwrap(); + let ptr = f.types.ptr_global(); + let entry = f.append_block(); + let p = f.append_block_param(entry, ptr); + let mem = MemFlags { + align: 3, + ..MemFlags::new() + }; + let v = f.append_load(entry, ty, p, mem); + f.set_terminator(entry, Terminator::Ret(Ret::one(v))); + f + } + + #[test] + fn natural_alignment_covers_scalars() { + assert_eq!(natural_align(0), 1); + assert_eq!(natural_align(1), 1); + assert_eq!(natural_align(3), 4); + assert_eq!(natural_align(8), 8); + assert_eq!(natural_align(100), 8); + assert_eq!(natural_align(u64::MAX), 8); + } + + #[test] + fn builders_fill_kind_size_and_lookup() { + let mut m = Module::new(); + m.add_function("f", constant(1)); + m.add_data("ro", vec![1, 2, 3]); + m.add_writable_relocs( + "rw", + vec![0; 8], + vec![DataReloc { + offset: 0, + symbol: "f".to_string(), + }], + ); + m.add_bss("zero", 64); + assert!(m.function("f").is_some()); + assert!(m.function("ro").is_none()); + let ro = m.global("ro").unwrap(); + assert_eq!((ro.kind, ro.size, ro.align), (DataKind::Rodata, 3, 0)); + assert_eq!(ro.effective_align(), 4); + let rw = m.global("rw").unwrap(); + assert_eq!((rw.kind, rw.size, rw.relocs.len()), (DataKind::Data, 8, 1)); + let zero = m.global("zero").unwrap(); + assert_eq!( + (zero.kind, zero.size, zero.bytes.len()), + (DataKind::Bss, 64, 0) + ); + assert_eq!(zero.effective_align(), 8); + assert!(m.global("f").is_none()); + assert_eq!(m.verify(), Ok(())); + } + + #[test] + fn explicit_alignment_overrides_natural() { + let mut m = Module::new(); + m.add_bss("big", 64); + m.globals[0].align = 64; + assert_eq!(m.globals[0].effective_align(), 64); + assert_eq!(m.verify(), Ok(())); + } + + #[test] + fn undefined_symbols_are_external_not_errors() { + let mut m = Module::new(); + m.add_data_relocs( + "table", + vec![0; 16], + vec![DataReloc { + offset: 8, + symbol: "elsewhere".to_string(), + }], + ); + assert_eq!(m.verify(), Ok(())); + } + + #[test] + fn rejects_duplicate_names_across_functions_and_globals() { + let mut m = Module::new(); + m.add_function("x", constant(1)); + m.add_data("x", vec![0]); + assert_eq!(m.verify(), Err(ModuleError::DuplicateName("x".to_string()))); + + let mut m = Module::new(); + m.add_function("f", constant(1)); + m.add_function("f", constant(2)); + assert_eq!(m.verify(), Err(ModuleError::DuplicateName("f".to_string()))); + + let mut m = Module::new(); + m.add_bss("b", 8); + m.add_writable("b", vec![0]); + assert_eq!(m.verify(), Err(ModuleError::DuplicateName("b".to_string()))); + } + + #[test] + fn rejects_size_mismatch_and_initialized_bss() { + let mut m = Module::new(); + m.add_data("d", vec![0; 4]); + m.globals[0].size = 8; + assert_eq!( + m.verify(), + Err(ModuleError::SizeMismatch { + name: "d".to_string(), + size: 8, + len: 4 + }) + ); + + let mut m = Module::new(); + m.add_bss("z", 8); + m.globals[0].bytes = vec![0; 8]; + assert_eq!( + m.verify(), + Err(ModuleError::BssInitialized("z".to_string())) + ); + + let mut m = Module::new(); + m.add_bss("z", 16); + m.globals[0].relocs.push(DataReloc { + offset: 0, + symbol: "f".to_string(), + }); + assert_eq!( + m.verify(), + Err(ModuleError::BssInitialized("z".to_string())) + ); + } + + #[test] + fn relocation_slot_must_fit_in_the_global() { + let reloc = |offset| DataReloc { + offset, + symbol: "s".to_string(), + }; + let mut m = Module::new(); + m.add_data_relocs("fits", vec![0; 16], vec![reloc(0), reloc(8)]); + assert_eq!(m.verify(), Ok(())); + + for offset in [9, 16, usize::MAX] { + let mut m = Module::new(); + m.add_data_relocs("d", vec![0; 16], vec![reloc(offset)]); + assert_eq!( + m.verify(), + Err(ModuleError::RelocOutOfRange { + name: "d".to_string(), + offset + }), + "offset {offset}" + ); + } + let mut m = Module::new(); + m.add_data_relocs("short", vec![0; 4], vec![reloc(0)]); + assert!(matches!( + m.verify(), + Err(ModuleError::RelocOutOfRange { .. }) + )); + } + + #[test] + fn alignment_must_be_zero_or_a_power_of_two() { + for align in [0, 1, 2, 16, 4096] { + let mut m = Module::new(); + m.add_data("d", vec![0; 4]); + m.globals[0].align = align; + assert_eq!(m.verify(), Ok(()), "align {align}"); + } + for align in [3, 6, 12] { + let mut m = Module::new(); + m.add_data("d", vec![0; 4]); + m.globals[0].align = align; + assert_eq!( + m.verify(), + Err(ModuleError::InvalidAlign { + name: "d".to_string(), + align + }), + "align {align}" + ); + } + } + + #[test] + fn function_diagnostics_surface_in_the_error() { + let mut m = Module::new(); + m.add_function("ok", constant(1)); + m.add_function("bad", malformed()); + match m.verify() { + Err(ModuleError::InvalidFunction { name, diagnostics }) => { + assert_eq!(name, "bad"); + assert_eq!(diagnostics.len(), 1); + assert!(matches!(diagnostics[0], Diagnostic::InvalidMemFlags(_))); + } + other => panic!("expected InvalidFunction, got {other:?}"), + } + } + + #[test] + fn errors_display_and_implement_error() { + fn is_error(_: &E) {} + let e = ModuleError::DuplicateName("x".to_string()); + is_error(&e); + assert_eq!(e.to_string(), "duplicate module symbol x"); + let mut m = Module::new(); + m.add_function("bad", malformed()); + let text = m.verify().unwrap_err().to_string(); + assert!(text.starts_with("function bad failed verification: InvalidMemFlags")); + } +} diff --git a/src/ir/src/parser.rs b/src/ir/src/parser.rs index 90dc628..6d5cf81 100644 --- a/src/ir/src/parser.rs +++ b/src/ir/src/parser.rs @@ -2,6 +2,75 @@ //! of the function printer. Values and blocks are named positionally (`v{n}`, //! `block{n}`); the parser creates them in textual order and resolves //! references by number. Ported from `vulcan/libs/vulcan-ir/parser.zig`. +//! +//! # Memory accesses +//! +//! `load` and `store` take optional modifier words (the `MemFlags`) between the mnemonic +//! and the operands. Each may appear at most once, in any order; the canonical order +//! (the one a printer writes) is: +//! +//! ```text +//! let v1 = load [volatile] [atomic ] [align ] [be|le] , +//! store [volatile] [atomic ] [align ] [be|le] , +//! ``` +//! +//! - `volatile`: the access may not be removed, merged or reordered. +//! - `atomic `: `relaxed`, `acquire`, `release`, `seq_cst` (loads take +//! relaxed/acquire/seq_cst, stores relaxed/release/seq_cst; `acq_rel` parses but does not +//! verify on a load or store). +//! - `align `: the address is aligned to `n` bytes (a power of two). +//! - `be` / `le`: big / little endian memory image; omitted means native. +//! +//! # Signedness lives on the operation +//! +//! Integer types are plain `iN` (there is no `uN`); an operation decides whether it reads its +//! operands as signed or unsigned. Binary operators are infix (`let v2 = v0 v1`, or +//! `v0 ` for the immediate form); the signed / unsigned variant of an +//! operator is the same spelling with an `s` / `u` suffix: +//! +//! ```text +//! + - * & | ^ << >> add sub mul and or xor shl, logical shift right (integer) +//! / % FLOAT division / remainder only +//! /u /s %u %s unsigned / signed integer division / remainder +//! >>s arithmetic shift right (integer) +//! *hu *hs high half of the unsigned / signed double-width product +//! == != equality of any scalar +//! < <= > >= FLOAT ordered comparison only +//! s >=s signed integer comparison +//! u >=u unsigned integer / pointer comparison +//! ``` +//! +//! Conversions name the kind first, then the result type: +//! +//! ```text +//! let v1 = convert , v0 +//! ``` +//! +//! with `kind` one of `trunc zext sext sitofp uitofp fptosi fptoui fpresize` (the legal type +//! pairs of each are listed on `ConvertKind`). A same-width integer cast does not exist. +//! +//! `dot` and the atomic min/max spell the signedness as a word: +//! +//! ```text +//! let v3 = dot signed|unsigned v0, v1, v2 +//! let v2 = atomic_rmw add|smin|smax|umin|umax|bit_and|bit_or|bit_xor|exchange v0, v1 +//! ``` +//! +//! Words naming a `BinOp` (`reduce add, v0`) are `add sub mul div rem udiv sdiv urem srem +//! bit_and bit_or bit_xor shl shr sar umulh smulh`. +//! +//! # Intrinsics +//! +//! An intrinsic is a target-defined operation named by a dotted identifier: +//! +//! ```text +//! let v2 = intrinsic @aarch64.aese(v0, v1) [imm ] [pure] +//! intrinsic @aarch64.dmb() [imm ] [pure] +//! ``` +//! +//! `imm` is the one signed immediate operand (omitted means 0, and the printer omits it +//! then). `pure` marks the intrinsic free of side effects (`Intrinsic::side_effects == +//! false`); without it the intrinsic is assumed to have side effects. use alloc::string::{String, ToString}; use alloc::vec::Vec; @@ -9,10 +78,10 @@ use alloc::vec::Vec; use super::attribute::{AttrValue, Attribute, Custom, Endianness}; use super::function::{ Alloca, Arith, AtomicOp, AtomicOrdering, AtomicRmw, AtomicScope, AttrTarget, BarrierScope, - BinOp, Block, Call, CallIndirect, CmpOp, Compare, Convert, EdgeDesc, Extract, Function, - GlobalAddr, InputSigns, Load, LowFloatConvert, MatMul, MatMulQuant, MatMulQuantOut, - MatMulScale, MatMulType, NvFp4Convert, Opcode, Reduce, Ret, RetPiece, Select, Terminator, - Unary, UnaryOp, Value, + BinOp, Block, Call, CallIndirect, CmpOp, Compare, ConvertKind, EdgeDesc, Extract, Function, + GlobalAddr, InputSigns, LowFloatConvert, MatMul, MatMulQuant, MatMulQuantOut, MatMulScale, + MatMulType, MemFlags, NvFp4Convert, Opcode, Reduce, Ret, RetPiece, Select, Terminator, Unary, + UnaryOp, Value, }; use super::low_float::Format as LowFloatFormat; use super::nvfp4::ScaleApplication; @@ -42,20 +111,45 @@ impl From for ParseError { } } -/// Map a leading character to a binary arithmetic operator, if it is one. -fn arith_op_of(c: u8) -> Option { - match c { - b'+' => Some(BinOp::Add), - b'-' => Some(BinOp::Sub), - b'*' => Some(BinOp::Mul), - b'/' => Some(BinOp::Div), - b'%' => Some(BinOp::Rem), - b'&' => Some(BinOp::BitAnd), - b'|' => Some(BinOp::BitOr), - b'^' => Some(BinOp::BitXor), - _ => None, - } -} +/// The infix spellings of the binary operators. A spelling that is a prefix of another +/// (`>>` of `>>s`, `*` of `*hu`) comes after it, so the first match wins. +const ARITH_OPS: [(&str, BinOp); 17] = [ + ("<<", BinOp::Shl), + (">>s", BinOp::Sar), + (">>", BinOp::Shr), + ("*hu", BinOp::UMulh), + ("*hs", BinOp::SMulh), + ("/u", BinOp::UDiv), + ("/s", BinOp::SDiv), + ("%u", BinOp::URem), + ("%s", BinOp::SRem), + ("+", BinOp::Add), + ("-", BinOp::Sub), + ("*", BinOp::Mul), + ("/", BinOp::Div), + ("%", BinOp::Rem), + ("&", BinOp::BitAnd), + ("|", BinOp::BitOr), + ("^", BinOp::BitXor), +]; + +/// The infix spellings of the comparisons, ordered like [`ARITH_OPS`]. +const CMP_OPS: [(&str, CmpOp); 14] = [ + ("==", CmpOp::Eq), + ("!=", CmpOp::Ne), + ("<=s", CmpOp::Sle), + ("<=u", CmpOp::Ule), + ("<=", CmpOp::Le), + ("=s", CmpOp::Sge), + (">=u", CmpOp::Uge), + (">=", CmpOp::Ge), + (">s", CmpOp::Sgt), + (">u", CmpOp::Ugt), + (">", CmpOp::Gt), +]; /// Parse a function from its text form. The returned function owns its memory. pub fn parse(text: &str) -> Result { @@ -132,7 +226,7 @@ fn all_digits(s: &[u8]) -> bool { !s.is_empty() && s.iter().all(|&c| is_digit(c)) } -/// BinOp from its printer word (`add`, `sub`, `bit_and`, ...). The text is untrusted, +/// BinOp from its word (`add`, `sub`, `bit_and`, `udiv`, ...). The text is untrusted, /// so an unknown op word is a parse error, never an invalid enum. fn bin_op_of(word: &str) -> Option { match word { @@ -141,12 +235,32 @@ fn bin_op_of(word: &str) -> Option { "mul" => Some(BinOp::Mul), "div" => Some(BinOp::Div), "rem" => Some(BinOp::Rem), + "udiv" => Some(BinOp::UDiv), + "sdiv" => Some(BinOp::SDiv), + "urem" => Some(BinOp::URem), + "srem" => Some(BinOp::SRem), "bit_and" => Some(BinOp::BitAnd), "bit_or" => Some(BinOp::BitOr), "bit_xor" => Some(BinOp::BitXor), "shl" => Some(BinOp::Shl), "shr" => Some(BinOp::Shr), - "mulh" => Some(BinOp::Mulh), + "sar" => Some(BinOp::Sar), + "umulh" => Some(BinOp::UMulh), + "smulh" => Some(BinOp::SMulh), + _ => None, + } +} + +fn convert_kind_of(word: &str) -> Option { + match word { + "trunc" => Some(ConvertKind::Trunc), + "zext" => Some(ConvertKind::Zext), + "sext" => Some(ConvertKind::Sext), + "sitofp" => Some(ConvertKind::SiToFp), + "uitofp" => Some(ConvertKind::UiToFp), + "fptosi" => Some(ConvertKind::FpToSi), + "fptoui" => Some(ConvertKind::FpToUi), + "fpresize" => Some(ConvertKind::FpResize), _ => None, } } @@ -209,8 +323,10 @@ fn barrier_scope_of(word: &str) -> Option { fn atomic_op_of(word: &str) -> Option { match word { "add" => Some(AtomicOp::Add), - "min" => Some(AtomicOp::Min), - "max" => Some(AtomicOp::Max), + "smin" => Some(AtomicOp::SMin), + "smax" => Some(AtomicOp::SMax), + "umin" => Some(AtomicOp::UMin), + "umax" => Some(AtomicOp::UMax), "bit_and" => Some(AtomicOp::BitAnd), "bit_or" => Some(AtomicOp::BitOr), "bit_xor" => Some(AtomicOp::BitXor), @@ -542,20 +658,6 @@ impl<'a> FunctionParser<'a> { let align = u32::try_from(n).map_err(|_| ParseError::InvalidSyntax)?; return Ok(Attribute::Align(align)); } - if word == "endian" { - self.eat(b'(')?; - let e = self.read_word().to_string(); - self.eat(b')')?; - // The text is UNTRUSTED, so an unknown order is a parse error and never an - // invalid enum. - let order = match e.as_str() { - "little" => Endianness::Little, - "big" => Endianness::Big, - "native" => Endianness::Native, - _ => return Err(ParseError::InvalidSyntax), - }; - return Ok(Attribute::Endian(order)); - } // Namespaced: `namespace.key` with an optional `= value`. A namespace may itself // hold dots (`vulcan.gpu` is the one the whole GPU path uses), so read the full // dotted path and split it at the LAST dot: everything before it is the @@ -715,6 +817,13 @@ impl<'a> FunctionParser<'a> { return Err(ParseError::InvalidSyntax); } self.parse_void_call(block)?; + } else if word == "intrinsic" { + if !pending.is_empty() { + return Err(ParseError::InvalidSyntax); + } + let (name, args, imm, side_effects) = self.parse_intrinsic_body()?; + self.func + .append_intrinsic_void(block, &name, &args, imm, side_effects); } else if word == "call_indirect" { if !pending.is_empty() { return Err(ParseError::InvalidSyntax); @@ -889,36 +998,24 @@ impl<'a> FunctionParser<'a> { .map_err(|_| ParseError::InvalidSyntax) } + /// Consume the binary arithmetic operator at the cursor, if there is one. + fn read_arith_op(&mut self) -> Option { + let rest = &self.src[self.pos..]; + let (text, op) = ARITH_OPS + .iter() + .find(|(text, _)| rest.starts_with(text.as_bytes()))?; + self.pos += text.len(); + Some(*op) + } + fn read_cmp_op(&mut self) -> Result { - match self.peek().ok_or(ParseError::InvalidSyntax)? { - b'=' => { - self.eat(b'=')?; - self.eat(b'=')?; - Ok(CmpOp::Eq) - } - b'!' => { - self.eat(b'!')?; - self.eat(b'=')?; - Ok(CmpOp::Ne) - } - b'<' => { - self.pos += 1; - Ok(if self.try_char(b'=') { - CmpOp::Le - } else { - CmpOp::Lt - }) - } - b'>' => { - self.pos += 1; - Ok(if self.try_char(b'=') { - CmpOp::Ge - } else { - CmpOp::Gt - }) - } - _ => Err(ParseError::InvalidSyntax), - } + let rest = &self.src[self.pos..]; + let (text, op) = CMP_OPS + .iter() + .find(|(text, _)| rest.starts_with(text.as_bytes())) + .ok_or(ParseError::InvalidSyntax)?; + self.pos += text.len(); + Ok(*op) } fn parse_let(&mut self, block: Block) -> Result { @@ -953,17 +1050,7 @@ impl<'a> FunctionParser<'a> { return Ok(result); } self.skip_ws(); - let rest = &self.src[self.pos..]; - if rest.starts_with(b"<<") { - self.pos += 2; - return self.finish_arith(block, lhs, BinOp::Shl); - } - if rest.starts_with(b">>") { - self.pos += 2; - return self.finish_arith(block, lhs, BinOp::Shr); - } - if let Some(bop) = arith_op_of(self.peek().unwrap_or(0)) { - self.pos += 1; + if let Some(bop) = self.read_arith_op() { return self.finish_arith(block, lhs, bop); } let op = self.read_cmp_op()?; @@ -978,21 +1065,24 @@ impl<'a> FunctionParser<'a> { } let op = self.read_word().to_string(); if op == "load" { - let is_volatile = self.try_word("volatile"); + let mem = self.parse_mem_flags()?; self.skip_ws(); let ty = self.parse_type()?; self.skip_ws(); self.eat(b',')?; self.skip_ws(); let ptr = self.parse_value_ref()?; - let result = self.func.append_inst( - block, - ty, - Opcode::Load(Load { - ptr, - volatile: is_volatile, - }), - ); + let result = self.func.append_load(block, ty, ptr, mem); + self.record_value(result); + return Ok(result); + } + if op == "intrinsic" { + self.skip_ws(); + let ty = self.parse_type()?; + let (name, args, imm, side_effects) = self.parse_intrinsic_body()?; + let result = self + .func + .append_intrinsic(block, ty, &name, &args, imm, side_effects); self.record_value(result); return Ok(result); } @@ -1018,6 +1108,12 @@ impl<'a> FunctionParser<'a> { return Ok(result); } if op == "dot" { + self.skip_ws(); + let signed = match self.read_word() { + "signed" => true, + "unsigned" => false, + _ => return Err(ParseError::InvalidSyntax), + }; self.skip_ws(); let acc = self.parse_value_ref()?; self.skip_ws(); @@ -1028,7 +1124,7 @@ impl<'a> FunctionParser<'a> { self.eat(b',')?; self.skip_ws(); let b = self.parse_value_ref()?; - let result = self.func.append_dot(block, acc, a, b); + let result = self.func.append_dot(block, acc, a, b, signed); self.record_value(result); return Ok(result); } @@ -1149,15 +1245,15 @@ impl<'a> FunctionParser<'a> { return Ok(result); } if op == "convert" { + self.skip_ws(); + let kind = convert_kind_of(self.read_word()).ok_or(ParseError::InvalidSyntax)?; self.skip_ws(); let ty = self.parse_type()?; self.skip_ws(); self.eat(b',')?; self.skip_ws(); let value = self.parse_value_ref()?; - let result = self - .func - .append_inst(block, ty, Opcode::Convert(Convert { value })); + let result = self.func.append_convert(block, ty, kind, value); self.record_value(result); return Ok(result); } @@ -1173,7 +1269,6 @@ impl<'a> FunctionParser<'a> { self.func.types.intern(TypeKind::Float(FloatKind::F32)) } else { self.func.types.intern(TypeKind::Int(IntDesc { - signed: false, bits: format.payload_bits(), })) }; @@ -1208,10 +1303,7 @@ impl<'a> FunctionParser<'a> { let ty = if dequantize { self.func.types.intern(TypeKind::Float(FloatKind::F32)) } else { - self.func.types.intern(TypeKind::Int(IntDesc { - signed: false, - bits: 8, - })) + self.func.types.intern(TypeKind::Int(IntDesc { bits: 8 })) }; let conversion = NvFp4Convert { value: operands[0], @@ -1290,18 +1382,107 @@ impl<'a> FunctionParser<'a> { }) } + /// Read `be` or `le` at the cursor; any other word leaves the cursor where it was. + fn try_endian_word(&mut self) -> Option { + let save = self.pos; + self.skip_ws(); + match self.read_word() { + "be" => Some(Endianness::Big), + "le" => Some(Endianness::Little), + _ => { + self.pos = save; + None + } + } + } + + /// Parse the optional modifier words after `load`/`store`: `volatile`, `atomic `, + /// `align `, `be`, `le`, in any order, each at most once. Absent modifiers keep the + /// `MemFlags::new()` default (native byte order, not atomic, no alignment promise). + fn parse_mem_flags(&mut self) -> Result { + let mut mem = MemFlags::new(); + let (mut volatile, mut atomic, mut align, mut endian) = (false, false, false, false); + loop { + if self.try_word("volatile") { + if core::mem::replace(&mut volatile, true) { + return Err(ParseError::InvalidSyntax); + } + mem.volatile = true; + } else if self.try_word("atomic") { + if core::mem::replace(&mut atomic, true) { + return Err(ParseError::InvalidSyntax); + } + self.skip_ws(); + mem.ordering = + Some(atomic_ordering_of(self.read_word()).ok_or(ParseError::InvalidSyntax)?); + } else if self.try_word("align") { + if core::mem::replace(&mut align, true) { + return Err(ParseError::InvalidSyntax); + } + self.skip_ws(); + mem.align = + u32::try_from(self.read_unsigned()?).map_err(|_| ParseError::InvalidSyntax)?; + } else if let Some(order) = self.try_endian_word() { + if core::mem::replace(&mut endian, true) { + return Err(ParseError::InvalidSyntax); + } + mem.endian = order; + } else { + return Ok(mem); + } + } + } + fn parse_store(&mut self, block: Block) -> Result<(), ParseError> { - let is_volatile = self.try_word("volatile"); + let mem = self.parse_mem_flags()?; self.skip_ws(); let value = self.parse_value_ref()?; self.skip_ws(); self.eat(b',')?; self.skip_ws(); let ptr = self.parse_value_ref()?; - self.func.append_store_vol(block, value, ptr, is_volatile); + self.func.append_store_mem(block, value, ptr, mem); Ok(()) } + /// The part of an intrinsic after its optional result type: + /// `@name(args) [imm N] [pure]`. The name is a dotted identifier (`aarch64.aese`); + /// `pure` clears `side_effects`, which is otherwise set. + fn parse_intrinsic_body(&mut self) -> Result<(String, Vec, i64, bool), ParseError> { + self.skip_ws(); + self.eat(b'@')?; + let start = self.pos; + while matches!(self.src.get(self.pos), Some(&c) if is_word_char(c) || c == b'.') { + self.pos += 1; + } + if self.pos == start { + return Err(ParseError::InvalidSyntax); + } + let name = core::str::from_utf8(&self.src[start..self.pos]) + .map_err(|_| ParseError::InvalidSyntax)? + .to_string(); + let mut args: Vec = Vec::new(); + self.parse_call_args(&mut args)?; + let (mut imm, mut side_effects) = (0, true); + let (mut saw_imm, mut saw_pure) = (false, false); + loop { + if self.try_word("imm") { + if core::mem::replace(&mut saw_imm, true) { + return Err(ParseError::InvalidSyntax); + } + self.skip_ws(); + imm = self.read_signed()?; + } else if self.try_word("pure") { + if core::mem::replace(&mut saw_pure, true) { + return Err(ParseError::InvalidSyntax); + } + side_effects = false; + } else { + return Ok((name, args, imm, side_effects)); + } + } + } + fn parse_barrier(&mut self, block: Block) -> Result<(), ParseError> { self.skip_ws(); let scope = barrier_scope_of(self.read_word()).ok_or(ParseError::InvalidSyntax)?; @@ -1839,11 +2020,472 @@ mod tests { #[test] fn round_trips_the_canonical_max_function() { let func = parse_ok( - "fn {\n block0(v0: i32, v1: i32):\n let v2 = v0 > v1\n if v2 { block1(v0) } else { block1(v1) }\n ret void\n\n block1(v3: i32):\n ret v3\n}", + "fn {\n block0(v0: i32, v1: i32):\n let v2 = v0 >s v1\n if v2 { block1(v0) } else { block1(v1) }\n ret void\n\n block1(v3: i32):\n ret v3\n}", ); let diags = verify::verify(&func, verify::Profile::High); assert!(diags.ok()); let diags = verify::verify(&func, verify::Profile::Low); assert!(diags.ok()); } + + fn only_mem(func: &Function, which: usize) -> MemFlags { + let mut seen = 0; + for bi in 0..func.block_count() { + for &inst in func.block_insts(Block::from_u32(bi as u32)) { + let mem = match func.opcode(inst) { + Opcode::Load(l) => l.mem, + Opcode::Store(st) => st.mem, + _ => continue, + }; + if seen == which { + return mem; + } + seen += 1; + } + } + panic!("no memory access #{which}"); + } + + /// The canonical spelling of `mem` as documented in the module comment. + fn mem_words(mem: &MemFlags) -> String { + let mut out = String::new(); + if mem.volatile { + out.push_str("volatile "); + } + if let Some(o) = mem.ordering { + out.push_str(match o { + AtomicOrdering::Relaxed => "atomic relaxed ", + AtomicOrdering::Acquire => "atomic acquire ", + AtomicOrdering::Release => "atomic release ", + AtomicOrdering::AcqRel => "atomic acq_rel ", + AtomicOrdering::SeqCst => "atomic seq_cst ", + }); + } + if mem.align != 0 { + out.push_str(&alloc::format!("align {} ", mem.align)); + } + match mem.endian { + Endianness::Native => {} + Endianness::Big => out.push_str("be "), + Endianness::Little => out.push_str("le "), + } + out + } + + #[test] + fn parses_load_and_store_without_modifiers_as_plain() { + let func = parse_ok( + "fn {\n block0(v0: ptr, v1: i32):\n let v2 = load i32, v0\n store v1, v0\n ret v2\n}", + ); + assert_eq!(only_mem(&func, 0), MemFlags::new()); + assert_eq!(only_mem(&func, 1), MemFlags::new()); + assert!(only_mem(&func, 0).is_plain()); + } + + #[test] + fn mem_flags_text_round_trips_every_modifier() { + let orderings = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Acquire), + Some(AtomicOrdering::Release), + Some(AtomicOrdering::AcqRel), + Some(AtomicOrdering::SeqCst), + ]; + let endians = [Endianness::Native, Endianness::Little, Endianness::Big]; + for volatile in [false, true] { + for align in [0u32, 1, 8, 4096] { + for ordering in orderings { + for endian in endians { + let mem = MemFlags { + volatile, + align, + ordering, + endian, + }; + let words = mem_words(&mem); + let text = alloc::format!( + "fn {{\n block0(v0: ptr, v1: i32):\n let v2 = load {words}i32, v0\n store {words}v1, v0\n ret v2\n}}" + ); + let func = parse_ok(&text); + assert_eq!(only_mem(&func, 0), mem, "{text}"); + assert_eq!(only_mem(&func, 1), mem, "{text}"); + } + } + } + } + } + + #[test] + fn mem_flag_modifiers_parse_in_any_order() { + let func = parse_ok( + "fn {\n block0(v0: ptr):\n let v1 = load be align 8 atomic seq_cst volatile i64, v0\n ret v1\n}", + ); + assert_eq!( + only_mem(&func, 0), + MemFlags { + volatile: true, + align: 8, + ordering: Some(AtomicOrdering::SeqCst), + endian: Endianness::Big, + } + ); + } + + #[test] + fn rejects_malformed_mem_flags() { + for words in [ + "volatile volatile ", + "align ", + "align x ", + "atomic ", + "atomic strong ", + "atomic acquire atomic acquire ", + "be le ", + "align 4 align 4 ", + ] { + let text = alloc::format!( + "fn {{\n block0(v0: ptr):\n let v1 = load {words}i32, v0\n ret v1\n}}" + ); + assert!(parse(&text).is_err(), "{text}"); + } + } + + #[test] + fn round_trips_a_valued_intrinsic() { + let func = parse_ok( + "fn {\n block0(v0: i32, v1: i32):\n let v2 = intrinsic i32 @aarch64.aese(v0, v1) imm -3\n ret v2\n}", + ); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + let Opcode::Intrinsic(i) = func.opcode(inst) else { + panic!("expected intrinsic"); + }; + assert_eq!(func.symbol_name(i.symbol), "aarch64.aese"); + assert_eq!( + func.value_list(i.args), + &[Value::from_u32(0), Value::from_u32(1)] + ); + assert_eq!(i.imm, -3); + assert!(i.side_effects); + } + + #[test] + fn round_trips_a_pure_and_a_void_intrinsic() { + let func = parse_ok( + "fn {\n block0(v0: i32):\n let v1 = intrinsic i32 @x86.bswap(v0) pure\n intrinsic @arm.dmb() imm 15\n intrinsic @nop.pure() pure imm 1\n ret v1\n}", + ); + let insts = func.block_insts(Block::from_u32(0)); + let get = |n: usize| match func.opcode(insts[n]) { + Opcode::Intrinsic(i) => i, + _ => panic!("expected intrinsic"), + }; + let (a, b, c) = (get(0), get(1), get(2)); + assert!(!a.side_effects && a.imm == 0); + assert!(func.inst_result(insts[1]).is_none()); + assert!(b.side_effects && b.imm == 15 && b.args.len == 0); + assert!(!c.side_effects && c.imm == 1); + assert_eq!(func.symbol_name(b.symbol), "arm.dmb"); + } + + // --- Signedness on the operation --------------------------------------------- + + const ALL_BIN_OPS: [BinOp; 17] = [ + BinOp::Add, + BinOp::Sub, + BinOp::Mul, + BinOp::Div, + BinOp::Rem, + BinOp::BitAnd, + BinOp::BitOr, + BinOp::BitXor, + BinOp::Shl, + BinOp::Shr, + BinOp::Sar, + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::UMulh, + BinOp::SMulh, + ]; + + const ALL_CMP_OPS: [CmpOp; 14] = [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Lt, + CmpOp::Le, + CmpOp::Gt, + CmpOp::Ge, + CmpOp::Slt, + CmpOp::Sle, + CmpOp::Sgt, + CmpOp::Sge, + CmpOp::Ult, + CmpOp::Ule, + CmpOp::Ugt, + CmpOp::Uge, + ]; + + fn assert_verifies(func: &Function, what: &str) { + let diags = verify::verify(func, verify::Profile::High); + assert!(diags.ok(), "{what}: {:?}", diags.items()); + } + + fn type_text(func: &Function, v: u32) -> String { + func.types + .display(func.value_type(Value::from_u32(v))) + .to_string() + } + + #[test] + fn every_binary_operator_parses_from_its_own_symbol() { + for op in ALL_BIN_OPS { + let ty = if matches!(op, BinOp::Div | BinOp::Rem) { + "f32" + } else { + "i32" + }; + let text = alloc::format!( + "fn {{\n block0(v0: {ty}, v1: {ty}):\n let v2 = v0 {} v1\n ret v2\n}}", + op.symbol() + ); + let func = parse_ok(&text); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::Arith(a) + if a.op == op && a.lhs == Value::from_u32(0) && a.rhs == Value::from_u32(1)), + "{op:?}: {text}" + ); + assert_verifies(&func, &text); + } + } + + #[test] + fn every_binary_operator_has_an_immediate_form() { + for op in ALL_BIN_OPS { + let ty = if matches!(op, BinOp::Div | BinOp::Rem) { + "f32" + } else { + "i32" + }; + let text = alloc::format!( + "fn {{\n block0(v0: {ty}):\n let v1 = v0 {} 3\n ret v1\n}}", + op.symbol() + ); + let func = parse_ok(&text); + let inst = func.defining_inst(Value::from_u32(1)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::ArithImm(a) if a.op == op && a.imm == 3), + "{op:?}: {text}" + ); + assert_verifies(&func, &text); + } + } + + #[test] + fn operator_suffixes_pick_the_signed_or_unsigned_variant() { + let op_of = |sym: &str| { + let func = parse_ok(&alloc::format!( + "fn {{\n block0(v0: i64, v1: i64):\n let v2 = v0 {sym} v1\n ret v2\n}}" + )); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + match func.opcode(inst) { + Opcode::Arith(a) => a.op, + other => panic!("expected arith, got {other:?}"), + } + }; + assert_eq!(op_of(">>"), BinOp::Shr); + assert_eq!(op_of(">>s"), BinOp::Sar); + assert_eq!(op_of("/u"), BinOp::UDiv); + assert_eq!(op_of("/s"), BinOp::SDiv); + assert_eq!(op_of("%u"), BinOp::URem); + assert_eq!(op_of("%s"), BinOp::SRem); + assert_eq!(op_of("*"), BinOp::Mul); + assert_eq!(op_of("*hu"), BinOp::UMulh); + assert_eq!(op_of("*hs"), BinOp::SMulh); + } + + #[test] + fn every_comparison_parses_from_its_own_symbol() { + for op in ALL_CMP_OPS { + let ty = if matches!(op, CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge) { + "f32" + } else { + "i32" + }; + let text = alloc::format!( + "fn {{\n block0(v0: {ty}, v1: {ty}):\n let v2 = v0 {} v1\n ret v2\n}}", + op.symbol() + ); + let func = parse_ok(&text); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::Icmp(c) if c.op == op), + "{op:?}: {text}" + ); + assert_eq!(type_text(&func, 2), "bool"); + assert_verifies(&func, &text); + } + let ptrs = + parse_ok("fn {\n block0(v0: ptr, v1: ptr):\n let v2 = v0 >"), + ("f32", ">>s"), + ("f32", "*hu"), + ("i32", "<"), + ("i32", ">="), + ("f32", "u"), + ("ptr", "", "<4 x i32>"), + ] { + let text = alloc::format!( + "fn {{\n block0(v0: {src}):\n let v1 = convert {} {dst}, v0\n ret v1\n}}", + kind.name() + ); + let func = parse_ok(&text); + let inst = func.defining_inst(Value::from_u32(1)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::Convert(c) + if c.kind == kind && c.value == Value::from_u32(0)), + "{text}" + ); + assert_eq!(type_text(&func, 1), dst, "{text}"); + assert_verifies(&func, &text); + } + } + + #[test] + fn a_conversion_needs_a_known_kind() { + for text in [ + "fn {\n block0(v0: i8):\n let v1 = convert i32, v0\n ret v1\n}", + "fn {\n block0(v0: i8):\n let v1 = convert widen i32, v0\n ret v1\n}", + "fn {\n block0(v0: i8):\n let v1 = convert sext, v0\n ret v1\n}", + ] { + assert!(parse(text).is_err(), "{text}"); + } + } + + #[test] + fn dot_spells_its_signedness() { + for (word, signed) in [("signed", true), ("unsigned", false)] { + let func = parse_ok(&alloc::format!( + "fn {{\n block0(v0: <4 x i32>, v1: <16 x i8>, v2: <16 x i8>):\n let v3 = dot {word} v0, v1, v2\n ret v3\n}}" + )); + let inst = func.defining_inst(Value::from_u32(3)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::Dot(d) + if d.signed == signed && d.acc == Value::from_u32(0) + && d.a == Value::from_u32(1) && d.b == Value::from_u32(2)), + "{word}" + ); + assert_eq!(type_text(&func, 3), "<4 x i32>"); + } + for text in [ + "fn {\n block0(v0: <4 x i32>, v1: <16 x i8>, v2: <16 x i8>):\n let v3 = dot v0, v1, v2\n ret v3\n}", + "fn {\n block0(v0: <4 x i32>, v1: <16 x i8>, v2: <16 x i8>):\n let v3 = dot sgn v0, v1, v2\n ret v3\n}", + ] { + assert!(parse(text).is_err(), "{text}"); + } + } + + #[test] + fn atomic_min_and_max_are_signed_or_unsigned() { + for (word, op) in [ + ("add", AtomicOp::Add), + ("smin", AtomicOp::SMin), + ("smax", AtomicOp::SMax), + ("umin", AtomicOp::UMin), + ("umax", AtomicOp::UMax), + ("bit_and", AtomicOp::BitAnd), + ("bit_or", AtomicOp::BitOr), + ("bit_xor", AtomicOp::BitXor), + ("exchange", AtomicOp::Exchange), + ] { + let func = parse_ok(&alloc::format!( + "fn {{\n block0(v0: ptr, v1: i32):\n let v2 = atomic_rmw {word} device seq_cst v0, v1\n ret v2\n}}" + )); + let inst = func.defining_inst(Value::from_u32(2)).unwrap(); + assert!( + matches!(func.opcode(inst), Opcode::AtomicRmw(a) if a.op == op), + "{word}" + ); + assert_verifies(&func, word); + } + for word in ["min", "max"] { + let text = alloc::format!( + "fn {{\n block0(v0: ptr, v1: i32):\n let v2 = atomic_rmw {word} device seq_cst v0, v1\n ret v2\n}}" + ); + assert!(parse(&text).is_err(), "{word}"); + } + } + + #[test] + fn integer_types_have_no_unsigned_spelling_and_mulh_has_no_bare_word() { + assert!(parse("fn {\n block0(v0: u32):\n ret v0\n}").is_err()); + assert!(parse("fn {\n block0():\n const v0: u8 = 1\n ret v0\n}").is_err()); + assert!(parse("fn {\n block0():\n const v0: i8 = 1\n ret v0\n}").is_ok()); + for word in ["mulh", "min", "max"] { + let text = alloc::format!( + "fn {{\n block0(v0: <4 x i32>):\n let v1 = reduce {word}, v0\n ret v1\n}}" + ); + assert!(parse(&text).is_err(), "{word}"); + } + for word in ["umulh", "sdiv", "sar"] { + let text = alloc::format!( + "fn {{\n block0(v0: <4 x i32>):\n let v1 = reduce {word}, v0\n ret v1\n}}" + ); + // The word parses; the verifier is what limits reductions to the lane-wise ops. + let func = parse_ok(&text); + assert!(!verify::verify(&func, verify::Profile::High).ok(), "{word}"); + } + } + + #[test] + fn low_float_and_nvfp4_results_are_plain_integers() { + let func = parse_ok( + "fn {\n block0(v0: f32, v1: i8, v2: f32):\n let v3 = encode_low_float bf16, v0\n let v4 = encode_low_float f8_e4m3, v0\n let v5 = quantize_nvfp4 multiply, divide, v0, v1, v2\n ret v3\n}", + ); + assert_eq!(type_text(&func, 3), "i16"); + assert_eq!(type_text(&func, 4), "i8"); + assert_eq!(type_text(&func, 5), "i8"); + } } diff --git a/src/ir/src/softfp.rs b/src/ir/src/softfp.rs index ec6e3dd..54443ed 100644 --- a/src/ir/src/softfp.rs +++ b/src/ir/src/softfp.rs @@ -2,7 +2,7 @@ extern crate alloc; use alloc::vec::Vec; -use super::function::{BinOp, Block, CmpOp, Function, Inst, Opcode, Value}; +use super::function::{BinOp, Block, CmpOp, Convert, ConvertKind, Function, Inst, Opcode, Value}; use super::types::Type; use super::types::{FloatKind, IntDesc, TypeKind}; @@ -52,17 +52,20 @@ fn float_kind(func: &Function, v: Value) -> Option { } } -struct IntInfo { - bits: u16, - signed: bool, +/// The width of the integer `v` has, if it is one. +fn int_bits(func: &Function, v: Value) -> Option { + match func.types.type_kind(func.value_type(v)) { + TypeKind::Int(i) => Some(i.bits), + _ => None, + } } -fn int_info(func: &Function, v: Value) -> Option { - match func.types.type_kind(func.value_type(v)) { - TypeKind::Int(i) => Some(IntInfo { - bits: i.bits, - signed: i.signed, - }), +/// The width of the integer a soft-float routine takes or returns for a `bits`-wide one: `int` +/// (32) up to 32 bits and `long long` (64) up to 64 bits; nothing wider has a routine. +fn routine_bits(bits: u16) -> Option { + match bits { + 1..=32 => Some(32), + 33..=64 => Some(64), _ => None, } } @@ -77,22 +80,22 @@ fn arith_sym(op: BinOp) -> Option<&'static str> { } } -fn cmp_sym(op: CmpOp) -> &'static str { +/// The comparison routine of a float compare and the SIGNED integer compare of its `int` +/// result against zero that gives the compare's value (the routines return `< 0`, `0`, `> 0`). +fn cmp_sym(op: CmpOp) -> Option<(&'static str, CmpOp)> { match op { - CmpOp::Eq => "__eqtf2", - CmpOp::Ne => "__netf2", - CmpOp::Lt => "__lttf2", - CmpOp::Le => "__letf2", - CmpOp::Gt => "__gttf2", - CmpOp::Ge => "__getf2", + CmpOp::Eq => Some(("__eqtf2", CmpOp::Eq)), + CmpOp::Ne => Some(("__netf2", CmpOp::Ne)), + CmpOp::Lt => Some(("__lttf2", CmpOp::Slt)), + CmpOp::Le => Some(("__letf2", CmpOp::Sle)), + CmpOp::Gt => Some(("__gttf2", CmpOp::Sgt)), + CmpOp::Ge => Some(("__getf2", CmpOp::Sge)), + _ => None, } } pub fn lower(func: &mut Function) -> bool { - let i32_ty = func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32_ty = func.types.intern(TypeKind::Int(IntDesc { bits: 32 })); let f128_ty = func.types.intern(TypeKind::Float(FloatKind::F128)); let mut changed = false; for bi in 0..func.block_count() { @@ -171,7 +174,10 @@ fn lower_inst( if !is_f128(func, c.lhs) { return false; } - let status = emit_call(func, order, i32_ty, cmp_sym(c.op), &[c.lhs, c.rhs]); + let Some((routine, against_zero)) = cmp_sym(c.op) else { + return false; + }; + let status = emit_call(func, order, i32_ty, routine, &[c.lhs, c.rhs]); let zero = emit_inst(func, order, i32_ty, Opcode::Iconst(0)); let res_ty = func.value_type(result); let cmp = emit_inst( @@ -179,7 +185,7 @@ fn lower_inst( order, res_ty, Opcode::Icmp(super::function::Compare { - op: c.op, + op: against_zero, lhs: status, rhs: zero, }), @@ -188,21 +194,94 @@ fn lower_inst( true } Opcode::Convert(cv) => { - let src_q = is_f128(func, cv.value); - let dst_q = is_f128(func, result); - if !src_q && !dst_q { - return false; - } - if src_q && dst_q { + let (src_q, dst_q) = (is_f128(func, cv.value), is_f128(func, result)); + // Neither side is f128, or both are (not a conversion): nothing to call. + if src_q == dst_q { return false; } let res_ty = func.value_type(result); - let sym = if dst_q { - to_f128_sym(func, cv.value) - } else { - from_f128_sym(func, cv.value, res_ty, func) + let call = match (cv.kind, dst_q) { + (ConvertKind::FpResize, true) => { + let routine = match float_kind(func, cv.value) { + Some(FloatKind::F32) => "__extendsftf2", + Some(FloatKind::F64) => "__extenddftf2", + Some(FloatKind::F16) => "__extendhftf2", + _ => return false, + }; + emit_call(func, order, res_ty, routine, &[cv.value]) + } + (ConvertKind::SiToFp | ConvertKind::UiToFp, true) => { + // The routines take an `int` / `long long`: a narrower source is widened + // first, sign- or zero-extending as the conversion reads it. + let signed = cv.kind == ConvertKind::SiToFp; + let Some(bits) = int_bits(func, cv.value) else { + return false; + }; + let Some(arg_bits) = routine_bits(bits) else { + return false; + }; + let arg_ty = func.types.intern(TypeKind::Int(IntDesc { bits: arg_bits })); + let arg = if bits < arg_bits { + let kind = if signed { + ConvertKind::Sext + } else { + ConvertKind::Zext + }; + let widen = Opcode::Convert(Convert { + value: cv.value, + kind, + }); + emit_inst(func, order, arg_ty, widen) + } else { + cv.value + }; + let routine = match (signed, arg_bits) { + (true, 32) => "__floatsitf", + (true, _) => "__floatditf", + (false, 32) => "__floatunsitf", + (false, _) => "__floatunditf", + }; + emit_call(func, order, res_ty, routine, &[arg]) + } + (ConvertKind::FpResize, false) => { + let routine = match float_kind(func, result) { + Some(FloatKind::F32) => "__trunctfsf2", + Some(FloatKind::F64) => "__trunctfdf2", + Some(FloatKind::F16) => "__trunctfhf2", + _ => return false, + }; + emit_call(func, order, res_ty, routine, &[cv.value]) + } + (ConvertKind::FpToSi | ConvertKind::FpToUi, false) => { + // The routine returns an `int` / `long long`; a narrower destination is + // that result truncated. + let signed = cv.kind == ConvertKind::FpToSi; + let Some(bits) = int_bits(func, result) else { + return false; + }; + let Some(ret_bits) = routine_bits(bits) else { + return false; + }; + let ret_ty = func.types.intern(TypeKind::Int(IntDesc { bits: ret_bits })); + let routine = match (signed, ret_bits) { + (true, 32) => "__fixtfsi", + (true, _) => "__fixtfdi", + (false, 32) => "__fixunstfsi", + (false, _) => "__fixunstfdi", + }; + let raw = emit_call(func, order, ret_ty, routine, &[cv.value]); + if bits < ret_bits { + let narrow = Opcode::Convert(Convert { + value: raw, + kind: ConvertKind::Trunc, + }); + emit_inst(func, order, res_ty, narrow) + } else { + raw + } + } + _ => return false, }; - let call = emit_call(func, order, res_ty, sym, &[cv.value]); func.replace_all_uses(result, call); true } @@ -218,48 +297,203 @@ fn lower_inst( } } -fn to_f128_sym(func: &Function, src: Value) -> &'static str { - if let Some(fk) = float_kind(func, src) { - return match fk { - FloatKind::F32 => "__extendsftf2", - FloatKind::F64 => "__extenddftf2", - FloatKind::F16 => "__extendhftf2", - FloatKind::F128 => unreachable!(), +#[cfg(test)] +mod tests { + use alloc::format; + use alloc::string::String; + use alloc::vec; + + use super::*; + use crate::function::{Compare, Ret, Terminator}; + use crate::verify::{Profile, verify}; + + /// One word per instruction of the entry block: the routine of a call, the kind of a + /// convert, the operator of a compare. + fn shape(func: &Function) -> Vec { + func.block_insts(Block::from_u32(0)) + .iter() + .map(|&inst| match func.opcode(inst) { + Opcode::Call(c) => format!("call {}", func.symbol_name(c.symbol)), + Opcode::Convert(c) => format!("convert {}", c.kind.name()), + Opcode::Icmp(c) => format!("icmp {}", c.op.symbol()), + Opcode::Iconst(_) => String::from("iconst"), + Opcode::Arith(a) => format!("arith {}", a.op.symbol()), + other => panic!("unexpected {other:?}"), + }) + .collect() + } + + /// `fn(x: src) -> dst { ret convert dst, x }` after `lower`. + fn lowered_convert(kind: ConvertKind, src: &str, dst: &str) -> (Function, bool) { + let mut func = Function::new(); + let (s, d) = ( + func.types.parse_type(src).unwrap(), + func.types.parse_type(dst).unwrap(), + ); + let entry = func.append_block(); + let x = func.append_block_param(entry, s); + let r = func.append_convert(entry, d, kind, x); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + let changed = lower(&mut func); + (func, changed) + } + + fn check(kind: ConvertKind, src: &str, dst: &str, want: &[&str]) { + let (func, changed) = lowered_convert(kind, src, dst); + assert!(changed, "{kind:?} {src} -> {dst}"); + assert_eq!(shape(&func), want, "{kind:?} {src} -> {dst}"); + let diags = verify(&func, Profile::High); + assert!(diags.ok(), "{kind:?} {src} -> {dst}: {:?}", diags.items()); + // The result the function returns is the last instruction's. + let Some(Terminator::Ret(ret)) = func.terminator(Block::from_u32(0)) else { + panic!("ret"); }; + let last = *func.block_insts(Block::from_u32(0)).last().unwrap(); + assert_eq!(func.inst_result(last), Some(ret.values[0])); + assert_eq!(func.value_type(ret.values[0]), { + let mut types = func.types.clone(); + types.parse_type(dst).unwrap() + }); + } + + #[test] + fn integers_narrower_than_the_routine_are_widened_as_the_conversion_reads_them() { + use ConvertKind::*; + check(SiToFp, "i8", "f128", &["convert sext", "call __floatsitf"]); + check(SiToFp, "i16", "f128", &["convert sext", "call __floatsitf"]); + check(SiToFp, "i32", "f128", &["call __floatsitf"]); + check(SiToFp, "i40", "f128", &["convert sext", "call __floatditf"]); + check(SiToFp, "i64", "f128", &["call __floatditf"]); + check( + UiToFp, + "i8", + "f128", + &["convert zext", "call __floatunsitf"], + ); + check(UiToFp, "i32", "f128", &["call __floatunsitf"]); + check( + UiToFp, + "i40", + "f128", + &["convert zext", "call __floatunditf"], + ); + check(UiToFp, "i64", "f128", &["call __floatunditf"]); + } + + #[test] + fn f128_to_integer_calls_the_routine_of_the_destination_sign_and_truncates() { + use ConvertKind::*; + check(FpToSi, "f128", "i32", &["call __fixtfsi"]); + check(FpToSi, "f128", "i8", &["call __fixtfsi", "convert trunc"]); + check(FpToSi, "f128", "i64", &["call __fixtfdi"]); + check(FpToSi, "f128", "i48", &["call __fixtfdi", "convert trunc"]); + check(FpToUi, "f128", "i32", &["call __fixunstfsi"]); + check( + FpToUi, + "f128", + "i16", + &["call __fixunstfsi", "convert trunc"], + ); + check(FpToUi, "f128", "i64", &["call __fixunstfdi"]); + } + + #[test] + fn f128_float_resizes_call_the_extend_and_truncate_routines() { + use ConvertKind::*; + check(FpResize, "f32", "f128", &["call __extendsftf2"]); + check(FpResize, "f64", "f128", &["call __extenddftf2"]); + check(FpResize, "f16", "f128", &["call __extendhftf2"]); + check(FpResize, "f128", "f32", &["call __trunctfsf2"]); + check(FpResize, "f128", "f64", &["call __trunctfdf2"]); + check(FpResize, "f128", "f16", &["call __trunctfhf2"]); } - let ii = int_info(func, src).expect("softfp convert source is a number"); - if ii.signed { - if ii.bits <= 32 { - "__floatsitf" - } else { - "__floatditf" + + #[test] + fn conversions_with_no_f128_or_no_routine_are_left_alone() { + use ConvertKind::*; + for (kind, src, dst) in [ + (SiToFp, "i32", "f32"), + (FpToUi, "f64", "i64"), + (FpResize, "f32", "f64"), + (Zext, "i8", "i32"), + // Wider than any routine's integer. + (SiToFp, "i128", "f128"), + (FpToSi, "f128", "i128"), + // Kind and types disagree (the verifier rejects these). + (Zext, "i8", "f128"), + (FpResize, "i32", "f128"), + ] { + let (func, changed) = lowered_convert(kind, src, dst); + assert!(!changed, "{kind:?} {src} -> {dst}"); + assert_eq!(shape(&func).len(), 1); } - } else if ii.bits <= 32 { - "__floatunsitf" - } else { - "__floatunditf" } -} -fn from_f128_sym(func: &Function, src: Value, dst_ty: Type, _f: &Function) -> &'static str { - let _ = src; - match func.types.type_kind(dst_ty) { - TypeKind::Float(FloatKind::F32) => "__trunctfsf2", - TypeKind::Float(FloatKind::F64) => "__trunctfdf2", - TypeKind::Float(FloatKind::F16) => "__trunctfhf2", - TypeKind::Int(i) => { - if i.signed { - if i.bits <= 32 { - "__fixtfsi" - } else { - "__fixtfdi" - } - } else if i.bits <= 32 { - "__fixunstfsi" - } else { - "__fixunstfdi" - } + #[test] + fn f128_comparisons_test_the_routines_int_result_against_zero_signed() { + for (op, routine, against_zero) in [ + (CmpOp::Eq, "__eqtf2", CmpOp::Eq), + (CmpOp::Ne, "__netf2", CmpOp::Ne), + (CmpOp::Lt, "__lttf2", CmpOp::Slt), + (CmpOp::Le, "__letf2", CmpOp::Sle), + (CmpOp::Gt, "__gttf2", CmpOp::Sgt), + (CmpOp::Ge, "__getf2", CmpOp::Sge), + ] { + let mut func = Function::new(); + let f128 = func.types.parse_type("f128").unwrap(); + let bool_t = func.types.parse_type("bool").unwrap(); + let entry = func.append_block(); + let a = func.append_block_param(entry, f128); + let b = func.append_block_param(entry, f128); + let r = func.append_inst(entry, bool_t, Opcode::Icmp(Compare { op, lhs: a, rhs: b })); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert!(lower(&mut func)); + assert_eq!( + shape(&func), + vec![ + format!("call {routine}"), + String::from("iconst"), + format!("icmp {}", against_zero.symbol()), + ] + ); + let diags = verify(&func, Profile::High); + assert!(diags.ok(), "{op:?}: {:?}", diags.items()); } - _ => unreachable!(), + } + + #[test] + fn f128_arithmetic_and_non_f128_instructions() { + let mut func = Function::new(); + let f128 = func.types.parse_type("f128").unwrap(); + let i32_t = func.types.parse_type("i32").unwrap(); + let entry = func.append_block(); + let a = func.append_block_param(entry, f128); + let b = func.append_block_param(entry, f128); + let x = func.append_block_param(entry, i32_t); + let y = func.append_block_param(entry, i32_t); + // Integer division is not an f128 operation and must not be touched. + let q = func.append_inst( + entry, + i32_t, + Opcode::Arith(crate::function::Arith { + op: BinOp::SDiv, + lhs: x, + rhs: y, + }), + ); + let s = func.append_inst( + entry, + f128, + Opcode::Arith(crate::function::Arith { + op: BinOp::Div, + lhs: a, + rhs: b, + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::many(&[q, s]))); + assert!(lower(&mut func)); + assert_eq!(shape(&func), ["arith /s", "call __divtf3"]); + let diags = verify(&func, Profile::High); + assert!(diags.ok(), "{:?}", diags.items()); } } diff --git a/src/ir/src/types.rs b/src/ir/src/types.rs index e8855b0..4cfe8db 100644 --- a/src/ir/src/types.rs +++ b/src/ir/src/types.rs @@ -7,12 +7,6 @@ use core::fmt; #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] pub struct Type(pub u32); -#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] -pub enum Signedness { - Signed, - Unsigned, -} - #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub enum FloatKind { F32, @@ -23,7 +17,6 @@ pub enum FloatKind { #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)] pub struct IntDesc { - pub signed: bool, pub bits: u16, } @@ -102,6 +95,20 @@ impl TypeTable { &self.kinds[h.0 as usize] } + /// Size in bytes of an int/float/ptr scalar as it sits in memory (pointers are 8 bytes); + /// None for bool, vectors, aggregates and integers whose width is not a whole byte count. + pub fn scalar_mem_bytes(&self, h: Type) -> Option { + match self.type_kind(h) { + TypeKind::Int(d) if d.bits % 8 == 0 && d.bits != 0 => Some(u32::from(d.bits) / 8), + TypeKind::Float(FloatKind::F16) => Some(2), + TypeKind::Float(FloatKind::F32) => Some(4), + TypeKind::Float(FloatKind::F64) => Some(8), + TypeKind::Float(FloatKind::F128) => Some(16), + TypeKind::Ptr(_) => Some(8), + _ => None, + } + } + pub fn count(&self) -> usize { self.kinds.len() } @@ -230,12 +237,9 @@ impl TypeTable { "f128" => Ok(self.intern(TypeKind::Float(FloatKind::F128))), _ => { let b = w.as_bytes(); - if b.len() >= 2 && (b[0] == b'i' || b[0] == b'u') { + if b.len() >= 2 && b[0] == b'i' { let bits: u16 = w[1..].parse().map_err(|_| ParseError)?; - Ok(self.intern(TypeKind::Int(IntDesc { - signed: b[0] == b'i', - bits, - }))) + Ok(self.intern(TypeKind::Int(IntDesc { bits }))) } else { Err(ParseError) } @@ -249,7 +253,7 @@ impl TypeTable { match &self.kinds[ty.0 as usize] { TypeKind::Bool => out.push_str("bool"), TypeKind::Int(i) => { - out.push(if i.signed { 'i' } else { 'u' }); + out.push('i'); out.push_str(&alloc::format!("{}", i.bits)); } TypeKind::Float(f) => out.push_str(match f { @@ -354,16 +358,24 @@ mod tests { #[test] fn scalars_round_trip() { let mut t = TypeTable::new(); - let i32a = t.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32a = t.intern(TypeKind::Int(IntDesc { bits: 32 })); assert_eq!(i32a, t.parse_type("i32").unwrap()); assert_eq!(t.parse_type("bool").unwrap(), t.intern(TypeKind::Bool)); assert_eq!(t.ptr_global(), t.parse_type("ptr").unwrap()); assert_eq!("i32", alloc::format!("{}", t.display(i32a))); } + #[test] + fn integer_types_are_signless_and_the_u_spelling_is_gone() { + let mut t = TypeTable::new(); + let i8t = t.parse_type("i8").unwrap(); + assert_eq!(i8t, t.intern(TypeKind::Int(IntDesc { bits: 8 }))); + assert_eq!(i8t, t.parse_type(" i8 ").unwrap()); + for text in ["u8", "u32", "u64", "i", "i-1", "i8x", "i99999"] { + assert!(t.parse_type(text).is_err(), "{text}"); + } + } + #[test] fn dedup_and_spaces() { let mut t = TypeTable::new(); @@ -378,10 +390,7 @@ mod tests { #[test] fn composites() { let mut t = TypeTable::new(); - let i32t = t.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })); + let i32t = t.intern(TypeKind::Int(IntDesc { bits: 32 })); let f32t = t.intern(TypeKind::Float(FloatKind::F32)); let v4 = t.intern(TypeKind::Vector(VectorDesc { len: 4, elem: i32t })); assert_eq!(v4, t.parse_type("<4 x i32>").unwrap()); diff --git a/src/ir/src/verify.rs b/src/ir/src/verify.rs index 38b97fb..b373c89 100644 --- a/src/ir/src/verify.rs +++ b/src/ir/src/verify.rs @@ -2,8 +2,11 @@ extern crate alloc; use alloc::vec::Vec; -use super::function::{AttrTarget, Function, Opcode, Value}; -use super::types::TypeKind; +use super::attribute::Endianness; +use super::function::{ + AtomicOrdering, BinOp, CmpOp, ConvertKind, Function, Inst, MemFlags, Opcode, Value, +}; +use super::types::{Type, TypeKind}; #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Profile { @@ -29,8 +32,12 @@ pub enum Diagnostic { value: Value, block: super::function::Block, }, - MisplacedEndian(AttrTarget), + InvalidMemFlags(super::function::Inst), OperandTypeMismatch(Value), + /// The operator is not defined for its operand (or result) types: integer division on + /// floats, a signed compare on pointers, an ill-formed conversion, and so on. Carries + /// the offending instruction. + InvalidOperation(Inst), } #[derive(Debug, Default)] @@ -56,8 +63,9 @@ pub fn verify(func: &Function, profile: Profile) -> Diagnostics { let mut diags = Diagnostics::default(); check_edges(func, &mut diags); check_dominance(func, &mut diags); - check_attributes(func, &mut diags); + check_mem_flags(func, &mut diags); check_operand_types(func, &mut diags); + check_operations(func, &mut diags); if profile == Profile::Low { check_primitive_types(func, &mut diags); } @@ -295,6 +303,7 @@ fn opcode_operands(func: &Function, inst: super::function::Inst) -> Vec { out.extend_from_slice(func.value_list(cf.then.args)); out.extend_from_slice(func.value_list(cf.else_.args)); } + Opcode::Intrinsic(i) => out.extend_from_slice(func.value_list(i.args)), Opcode::Iconst(_) | Opcode::Fconst(_) | Opcode::Fconst128(_) @@ -305,30 +314,62 @@ fn opcode_operands(func: &Function, inst: super::function::Inst) -> Vec { out } -fn is_memory_op(op: &Opcode) -> bool { - matches!( - op, - Opcode::Load(_) | Opcode::Store(_) | Opcode::Prefetch(_) | Opcode::Matmul(_) - ) -} - -fn check_attributes(func: &Function, diags: &mut Diagnostics) { - for entry in func.attribute_entries() { - if let super::attribute::Attribute::Endian(_) = entry.attr { - let ok = match entry.target { - AttrTarget::Value(v) => func - .defining_inst(v) - .is_some_and(|i| is_memory_op(&func.opcode(i))), - AttrTarget::Inst(i) => is_memory_op(&func.opcode(i)), - AttrTarget::Func | AttrTarget::Block(_) => false, +/// Checks the `MemFlags` of every load and store against the accessed type. +fn check_mem_flags(func: &Function, diags: &mut Diagnostics) { + for bi in 0..func.block_count() { + let b = super::function::Block::from_u32(bi as u32); + for &inst in func.block_insts(b) { + let (mem, ty, is_store) = match func.opcode_ref(inst) { + Opcode::Load(l) => match func.inst_result(inst) { + Some(r) => (l.mem, func.value_type(r), false), + None => { + diags.list.push(Diagnostic::InvalidMemFlags(inst)); + continue; + } + }, + Opcode::Store(st) => (st.mem, func.value_type(st.value), true), + _ => continue, }; - if !ok { - diags.list.push(Diagnostic::MisplacedEndian(entry.target)); + if !mem_flags_valid(func, &mem, ty, is_store) { + diags.list.push(Diagnostic::InvalidMemFlags(inst)); } } } } +/// Whether `mem` is a legal flag set for a load (`is_store == false`) or store of `ty`. +pub(crate) fn mem_flags_valid(func: &Function, mem: &MemFlags, ty: Type, is_store: bool) -> bool { + if mem.align != 0 && !mem.align.is_power_of_two() { + return false; + } + let bytes = func.types.scalar_mem_bytes(ty); + if let Some(ordering) = mem.ordering { + let allowed = if is_store { + matches!( + ordering, + AtomicOrdering::Relaxed | AtomicOrdering::Release | AtomicOrdering::SeqCst + ) + } else { + matches!( + ordering, + AtomicOrdering::Relaxed | AtomicOrdering::Acquire | AtomicOrdering::SeqCst + ) + }; + let sized = bytes.is_some_and(|n| n.is_power_of_two() && n <= 8); + if !allowed || !sized { + return false; + } + } + if mem.endian != Endianness::Little && mem.endian != Endianness::Native { + // A byte swap only has a meaning on a 1, 2, 4 or 8 byte int/float/ptr scalar + // (one byte is a no-op). + if !bytes.is_some_and(|n| matches!(n, 1 | 2 | 4 | 8)) { + return false; + } + } + true +} + fn is_ptr(func: &Function, v: Value) -> bool { matches!(func.types.type_kind(func.value_type(v)), TypeKind::Ptr(_)) } @@ -412,6 +453,158 @@ fn check_operand_types(func: &Function, diags: &mut Diagnostics) { } } +/// The kind of a scalar type, or of the lanes of a vector type. +fn lane_kind(func: &Function, ty: Type) -> &TypeKind { + match func.types.type_kind(ty) { + TypeKind::Vector(v) => func.types.type_kind(v.elem), + other => other, + } +} + +/// The lane count of a vector type; `None` for a scalar. +fn lane_count(func: &Function, ty: Type) -> Option { + match func.types.type_kind(ty) { + TypeKind::Vector(v) => Some(v.len), + _ => None, + } +} + +/// The width in bits of an integer (or `bool`, one bit) lane. +fn lane_bits(kind: &TypeKind) -> Option { + match kind { + TypeKind::Int(i) => Some(i.bits), + TypeKind::Bool => Some(1), + _ => None, + } +} + +fn lane_is_int(kind: &TypeKind) -> bool { + matches!(kind, TypeKind::Int(_)) +} + +fn lane_is_float(kind: &TypeKind) -> bool { + matches!(kind, TypeKind::Float(_)) +} + +/// Whether `op` is defined for operands whose lanes are `lane`. +fn binop_accepts(op: BinOp, lane: &TypeKind) -> bool { + match op { + BinOp::Add | BinOp::Sub => { + lane_is_int(lane) || lane_is_float(lane) || matches!(lane, TypeKind::Ptr(_)) + } + BinOp::Mul => lane_is_int(lane) || lane_is_float(lane), + BinOp::Div | BinOp::Rem => lane_is_float(lane), + BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => { + lane_is_int(lane) || matches!(lane, TypeKind::Bool) + } + BinOp::Shl + | BinOp::Shr + | BinOp::Sar + | BinOp::UDiv + | BinOp::SDiv + | BinOp::URem + | BinOp::SRem + | BinOp::UMulh + | BinOp::SMulh => lane_is_int(lane), + } +} + +/// Whether `op` is defined for operands whose lanes are `lane`. +fn cmpop_accepts(op: CmpOp, lane: &TypeKind) -> bool { + match op { + CmpOp::Eq | CmpOp::Ne => matches!( + lane, + TypeKind::Bool | TypeKind::Int(_) | TypeKind::Float(_) | TypeKind::Ptr(_) + ), + CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge => lane_is_float(lane), + CmpOp::Slt | CmpOp::Sle | CmpOp::Sgt | CmpOp::Sge => lane_is_int(lane), + CmpOp::Ult | CmpOp::Ule | CmpOp::Ugt | CmpOp::Uge => { + lane_is_int(lane) || matches!(lane, TypeKind::Ptr(_)) + } + } +} + +/// Whether converting `src` to `dst` is the conversion `kind` (see [`ConvertKind`]). +fn convert_ok(func: &Function, kind: ConvertKind, src: Type, dst: Type) -> bool { + if lane_count(func, src) != lane_count(func, dst) { + return false; + } + let (s, d) = (lane_kind(func, src), lane_kind(func, dst)); + match kind { + ConvertKind::Trunc => { + lane_is_int(s) + && lane_is_int(d) + && lane_bits(d).unwrap_or(0) < lane_bits(s).unwrap_or(0) + } + ConvertKind::Zext => { + lane_bits(s).is_some() + && lane_is_int(d) + && lane_bits(d).unwrap_or(0) > lane_bits(s).unwrap_or(0) + } + ConvertKind::Sext => { + lane_is_int(s) + && lane_is_int(d) + && lane_bits(d).unwrap_or(0) > lane_bits(s).unwrap_or(0) + } + ConvertKind::SiToFp | ConvertKind::UiToFp => lane_is_int(s) && lane_is_float(d), + ConvertKind::FpToSi | ConvertKind::FpToUi => lane_is_float(s) && lane_is_int(d), + ConvertKind::FpResize => lane_is_float(s) && lane_is_float(d) && s != d, + } +} + +/// Whether the arithmetic `op` over `lhs` (and `rhs`) producing `result` is well typed. +/// Pointer arithmetic (`ptr +/- int`, `int + ptr`) mixes types, so only the operator rule is +/// checked for it; otherwise the result has the operand type. +fn arith_ok( + func: &Function, + op: BinOp, + lhs: Value, + rhs: Option, + result: Option, +) -> bool { + let lt = func.value_type(lhs); + let lane = lane_kind(func, lt); + if !binop_accepts(op, lane) { + return false; + } + let mixed = rhs.is_some_and(|r| func.value_type(r) != lt); + if mixed || matches!(lane, TypeKind::Ptr(_)) { + return true; + } + result.is_none_or(|r| func.value_type(r) == lt) +} + +/// Checks that every operator is defined for the types it is applied to: the signed / +/// unsigned / floating-point operator families (`BinOp`, `CmpOp`) and the legal type pairs +/// of each `ConvertKind`. +fn check_operations(func: &Function, diags: &mut Diagnostics) { + for bi in 0..func.block_count() { + let b = super::function::Block::from_u32(bi as u32); + for &inst in func.block_insts(b) { + let result = func.inst_result(inst); + let ok = match func.opcode_ref(inst) { + Opcode::Arith(a) => arith_ok(func, a.op, a.lhs, Some(a.rhs), result), + Opcode::ArithImm(a) => arith_ok(func, a.op, a.lhs, None, result), + Opcode::Icmp(c) => { + let lt = func.value_type(c.lhs); + let scalar_result = result.is_none_or(|r| { + lane_count(func, lt).is_some() + || matches!(func.types.type_kind(func.value_type(r)), TypeKind::Bool) + }); + cmpop_accepts(c.op, lane_kind(func, lt)) && scalar_result + } + Opcode::Convert(cv) => result.is_some_and(|r| { + convert_ok(func, cv.kind, func.value_type(cv.value), func.value_type(r)) + }), + _ => true, + }; + if !ok { + diags.list.push(Diagnostic::InvalidOperation(inst)); + } + } + } +} + fn pointer_arith(func: &Function, lhs: Value, rhs: Value, op: super::function::BinOp) -> bool { if !matches!( op, @@ -439,3 +632,572 @@ fn check_primitive_types(func: &Function, diags: &mut Diagnostics) { } } } + +#[cfg(test)] +mod tests { + use alloc::vec; + + use super::*; + use crate::function::{Arith, Compare, Ret, Terminator}; + use crate::types::{FloatKind, IntDesc, VectorDesc}; + + fn mem(ordering: Option, endian: Endianness, align: u32) -> MemFlags { + MemFlags { + volatile: false, + align, + ordering, + endian, + } + } + + fn int(bits: u16) -> TypeKind { + TypeKind::Int(IntDesc { bits }) + } + + /// Number of `InvalidMemFlags` diagnostics for a load and for a store of `kind` with `m`. + fn bad_counts(kind: TypeKind, m: MemFlags) -> (usize, usize) { + let count = |is_store: bool| { + let mut func = Function::new(); + let ty = func.types.intern(kind.clone()); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + if is_store { + let v = func.append_block_param(entry, ty); + func.append_store_mem(entry, v, p, m); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + } else { + let v = func.append_load(entry, ty, p, m); + func.set_terminator(entry, Terminator::Ret(Ret::one(v))); + } + let diags = verify(&func, Profile::High); + assert!( + diags + .items() + .iter() + .all(|d| matches!(d, Diagnostic::InvalidMemFlags(_))), + "{:?}", + diags.items() + ); + diags.count() + }; + (count(false), count(true)) + } + + fn ok(kind: TypeKind, m: MemFlags) -> bool { + bad_counts(kind, m) == (0, 0) + } + + const NATIVE: Endianness = Endianness::Native; + + #[test] + fn default_and_plain_flags_verify() { + assert!(ok(int(32), MemFlags::new())); + assert!(ok(TypeKind::Bool, MemFlags::new())); + assert!(ok( + TypeKind::Struct(vec![]), + mem(None, Endianness::Little, 0) + )); + let volatile = MemFlags { + volatile: true, + ..MemFlags::new() + }; + assert!(ok(int(8), volatile)); + } + + #[test] + fn rejects_an_ordering_invalid_for_the_access_kind() { + use AtomicOrdering::*; + for o in [Relaxed, Acquire, SeqCst] { + let (load, _) = bad_counts(int(32), mem(Some(o), NATIVE, 0)); + assert_eq!(load, 0, "load {o:?}"); + } + for o in [Release, AcqRel] { + let (load, _) = bad_counts(int(32), mem(Some(o), NATIVE, 0)); + assert_eq!(load, 1, "load {o:?}"); + } + for o in [Relaxed, Release, SeqCst] { + let (_, store) = bad_counts(int(32), mem(Some(o), NATIVE, 0)); + assert_eq!(store, 0, "store {o:?}"); + } + for o in [Acquire, AcqRel] { + let (_, store) = bad_counts(int(32), mem(Some(o), NATIVE, 0)); + assert_eq!(store, 1, "store {o:?}"); + } + } + + #[test] + fn atomic_accesses_need_a_power_of_two_scalar_of_at_most_8_bytes() { + let seq = Some(AtomicOrdering::SeqCst); + for bits in [8, 16, 32, 64] { + assert!(ok(int(bits), mem(seq, NATIVE, 0)), "i{bits}"); + } + assert!(ok(TypeKind::Float(FloatKind::F32), mem(seq, NATIVE, 0))); + assert!(ok(TypeKind::Float(FloatKind::F64), mem(seq, NATIVE, 0))); + assert!(ok(TypeKind::Float(FloatKind::F16), mem(seq, NATIVE, 0))); + assert!(ok( + TypeKind::Ptr(crate::types::AddressSpace::Global), + mem(seq, NATIVE, 0) + )); + assert_eq!(bad_counts(int(128), mem(seq, NATIVE, 0)), (1, 1), "i128"); + assert_eq!(bad_counts(int(24), mem(seq, NATIVE, 0)), (1, 1), "i24"); + assert_eq!(bad_counts(int(1), mem(seq, NATIVE, 0)), (1, 1), "i1"); + assert_eq!(bad_counts(TypeKind::Bool, mem(seq, NATIVE, 0)), (1, 1)); + assert_eq!( + bad_counts(TypeKind::Float(FloatKind::F128), mem(seq, NATIVE, 0)), + (1, 1) + ); + assert_eq!( + bad_counts(TypeKind::Struct(vec![]), mem(seq, NATIVE, 0)), + (1, 1) + ); + } + + #[test] + fn big_endian_needs_a_1_2_4_or_8_byte_scalar() { + let big = Endianness::Big; + for bits in [8, 16, 32, 64] { + assert!(ok(int(bits), mem(None, big, 0)), "i{bits}"); + } + assert!(ok(TypeKind::Float(FloatKind::F32), mem(None, big, 0))); + assert!(ok(TypeKind::Float(FloatKind::F16), mem(None, big, 0))); + assert!(ok( + TypeKind::Ptr(crate::types::AddressSpace::Shared), + mem(None, big, 0) + )); + assert_eq!(bad_counts(int(128), mem(None, big, 0)), (1, 1)); + assert_eq!(bad_counts(int(24), mem(None, big, 0)), (1, 1)); + assert_eq!(bad_counts(TypeKind::Bool, mem(None, big, 0)), (1, 1)); + assert_eq!( + bad_counts(TypeKind::Float(FloatKind::F128), mem(None, big, 0)), + (1, 1) + ); + assert_eq!( + bad_counts(TypeKind::Struct(vec![]), mem(None, big, 0)), + (1, 1) + ); + let mut func = Function::new(); + let elem = func.types.intern(int(32)); + let vector = TypeKind::Vector(VectorDesc { len: 4, elem }); + assert_eq!(bad_counts(vector.clone(), mem(None, big, 0)), (1, 1)); + assert!(ok(vector, mem(None, Endianness::Little, 0))); + } + + #[test] + fn rejects_an_align_that_is_not_a_power_of_two() { + for align in [0, 1, 2, 16, 4096] { + assert!(ok(int(32), mem(None, NATIVE, align)), "align {align}"); + } + for align in [3, 6, 12, 4095] { + assert_eq!( + bad_counts(int(32), mem(None, NATIVE, align)), + (1, 1), + "align {align}" + ); + } + } + + #[test] + fn a_load_with_no_result_is_rejected() { + let mut func = Function::new(); + let ptr_t = func.types.ptr_global(); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + func.append_stmt_raw( + entry, + Opcode::Load(crate::function::Load { + ptr: p, + mem: MemFlags::new(), + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + assert_eq!(verify(&func, Profile::High).count(), 1); + } + + #[test] + fn intrinsic_operands_are_dominance_checked() { + let mut func = Function::new(); + let i32_t = func.types.intern(int(32)); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let r = func.append_intrinsic(entry, i32_t, "t.op", &[a], 1, false); + func.append_intrinsic_void(entry, "t.sink", &[r, a], 0, true); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert!(verify(&func, Profile::High).ok()); + } + + // --- Operator / type rules --------------------------------------------------- + + /// The type shapes the operator tables are checked against; vectors have 4 lanes. + #[derive(Clone, Copy, Debug)] + enum K { + Bool, + I(u16), + F(FloatKind), + Ptr, + VI(u16), + VF(FloatKind), + } + + fn make(func: &mut Function, k: K) -> Type { + let scalar = |func: &mut Function, kind: TypeKind| func.types.intern(kind); + match k { + K::Bool => scalar(func, TypeKind::Bool), + K::I(bits) => scalar(func, int(bits)), + K::F(f) => scalar(func, TypeKind::Float(f)), + K::Ptr => func.types.ptr_global(), + K::VI(bits) => { + let elem = scalar(func, int(bits)); + scalar(func, TypeKind::Vector(VectorDesc { len: 4, elem })) + } + K::VF(f) => { + let elem = scalar(func, TypeKind::Float(f)); + scalar(func, TypeKind::Vector(VectorDesc { len: 4, elem })) + } + } + } + + /// Asserts every diagnostic is an `InvalidOperation` and returns how many there are. + fn invalid_operations(func: &Function) -> usize { + let diags = verify(func, Profile::High); + assert!( + diags + .items() + .iter() + .all(|d| matches!(d, Diagnostic::InvalidOperation(_))), + "{:?}", + diags.items() + ); + diags.count() + } + + /// `fn(a: k, b: k) { ret a b }`. + fn arith_bad(k: K, op: BinOp) -> usize { + let mut func = Function::new(); + let ty = make(&mut func, k); + let entry = func.append_block(); + let a = func.append_block_param(entry, ty); + let b = func.append_block_param(entry, ty); + let r = func.append_inst(entry, ty, Opcode::Arith(Arith { op, lhs: a, rhs: b })); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + invalid_operations(&func) + } + + /// `fn(a: k) { ret a 1 }`. + fn arith_imm_bad(k: K, op: BinOp) -> usize { + let mut func = Function::new(); + let ty = make(&mut func, k); + let entry = func.append_block(); + let a = func.append_block_param(entry, ty); + let r = func.append_arith_imm(entry, ty, op, a, 1); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + invalid_operations(&func) + } + + /// `fn(a: k, b: k) { ret a b }` for a comparison; the result is `bool` (a `bool` + /// vector for vector operands). + fn cmp_bad(k: K, op: CmpOp) -> usize { + let mut func = Function::new(); + let ty = make(&mut func, k); + let bool_t = func.types.intern(TypeKind::Bool); + let result_t = match k { + K::VI(_) | K::VF(_) => func.types.intern(TypeKind::Vector(VectorDesc { + len: 4, + elem: bool_t, + })), + _ => bool_t, + }; + let entry = func.append_block(); + let a = func.append_block_param(entry, ty); + let b = func.append_block_param(entry, ty); + let r = func.append_inst( + entry, + result_t, + Opcode::Icmp(Compare { op, lhs: a, rhs: b }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + invalid_operations(&func) + } + + /// `fn(a: src) { ret convert dst, a }`. + fn convert_bad(kind: ConvertKind, src: K, dst: K) -> usize { + let mut func = Function::new(); + let (s, d) = (make(&mut func, src), make(&mut func, dst)); + let entry = func.append_block(); + let a = func.append_block_param(entry, s); + let r = func.append_convert(entry, d, kind, a); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + invalid_operations(&func) + } + + const F32: K = K::F(FloatKind::F32); + const F64: K = K::F(FloatKind::F64); + + #[test] + fn arithmetic_operators_are_split_by_integer_and_float_domain() { + use BinOp::*; + // (operator, defined on integers, defined on floats) + let table = [ + (Add, true, true), + (Sub, true, true), + (Mul, true, true), + (Div, false, true), + (Rem, false, true), + (UDiv, true, false), + (SDiv, true, false), + (URem, true, false), + (SRem, true, false), + (Shl, true, false), + (Shr, true, false), + (Sar, true, false), + (UMulh, true, false), + (SMulh, true, false), + (BitAnd, true, false), + (BitOr, true, false), + (BitXor, true, false), + ]; + for (op, on_int, on_float) in table { + for k in [K::I(8), K::I(32), K::I(64), K::VI(32)] { + assert_eq!(arith_bad(k, op), usize::from(!on_int), "{op:?} on {k:?}"); + assert_eq!( + arith_imm_bad(k, op), + usize::from(!on_int), + "{op:?} imm {k:?}" + ); + } + for k in [F32, F64, K::F(FloatKind::F16), K::VF(FloatKind::F32)] { + assert_eq!(arith_bad(k, op), usize::from(!on_float), "{op:?} on {k:?}"); + assert_eq!( + arith_imm_bad(k, op), + usize::from(!on_float), + "{op:?} imm {k:?}" + ); + } + } + } + + #[test] + fn bitwise_operators_also_accept_bool_but_nothing_else_does() { + for op in [BinOp::BitAnd, BinOp::BitOr, BinOp::BitXor] { + assert_eq!(arith_bad(K::Bool, op), 0, "{op:?}"); + } + for op in [BinOp::Add, BinOp::Mul, BinOp::Shl, BinOp::UDiv, BinOp::Div] { + assert_eq!(arith_bad(K::Bool, op), 1, "{op:?} on bool"); + } + for op in [BinOp::Mul, BinOp::Shr, BinOp::SDiv, BinOp::BitAnd] { + assert_eq!(arith_bad(K::Ptr, op), 1, "{op:?} on ptr"); + } + } + + #[test] + fn pointer_arithmetic_still_mixes_pointer_and_integer_operands() { + let mut func = Function::new(); + let ptr_t = func.types.ptr_global(); + let i64_t = func.types.intern(int(64)); + let entry = func.append_block(); + let p = func.append_block_param(entry, ptr_t); + let n = func.append_block_param(entry, i64_t); + let q = func.append_inst( + entry, + ptr_t, + Opcode::Arith(Arith { + op: BinOp::Add, + lhs: p, + rhs: n, + }), + ); + let r = func.append_arith_imm(entry, ptr_t, BinOp::Sub, q, 8); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert!(verify(&func, Profile::High).ok()); + } + + #[test] + fn an_arithmetic_result_must_have_the_operand_type() { + let mut func = Function::new(); + let i32_t = func.types.intern(int(32)); + let i64_t = func.types.intern(int(64)); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let b = func.append_block_param(entry, i32_t); + let r = func.append_inst( + entry, + i64_t, + Opcode::Arith(Arith { + op: BinOp::Add, + lhs: a, + rhs: b, + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert_eq!(invalid_operations(&func), 1); + } + + #[test] + fn comparison_operators_are_split_by_domain() { + use CmpOp::*; + // (operator, ints, floats, bools, pointers) + let table = [ + (Eq, true, true, true, true), + (Ne, true, true, true, true), + (Lt, false, true, false, false), + (Le, false, true, false, false), + (Gt, false, true, false, false), + (Ge, false, true, false, false), + (Slt, true, false, false, false), + (Sle, true, false, false, false), + (Sgt, true, false, false, false), + (Sge, true, false, false, false), + (Ult, true, false, false, true), + (Ule, true, false, false, true), + (Ugt, true, false, false, true), + (Uge, true, false, false, true), + ]; + for (op, on_int, on_float, on_bool, on_ptr) in table { + for k in [K::I(8), K::I(32), K::I(64), K::VI(32)] { + assert_eq!(cmp_bad(k, op), usize::from(!on_int), "{op:?} on {k:?}"); + } + for k in [F32, F64, K::VF(FloatKind::F32)] { + assert_eq!(cmp_bad(k, op), usize::from(!on_float), "{op:?} on {k:?}"); + } + assert_eq!( + cmp_bad(K::Bool, op), + usize::from(!on_bool), + "{op:?} on bool" + ); + assert_eq!(cmp_bad(K::Ptr, op), usize::from(!on_ptr), "{op:?} on ptr"); + } + } + + #[test] + fn a_scalar_comparison_yields_bool() { + let mut func = Function::new(); + let i32_t = func.types.intern(int(32)); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + let b = func.append_block_param(entry, i32_t); + let r = func.append_inst( + entry, + i32_t, + Opcode::Icmp(Compare { + op: CmpOp::Ult, + lhs: a, + rhs: b, + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::one(r))); + assert_eq!(invalid_operations(&func), 1); + } + + #[test] + fn conversions_accept_exactly_their_type_pairs() { + use ConvertKind::*; + let (i8, i16, i32, i64) = (K::I(8), K::I(16), K::I(32), K::I(64)); + let f16 = K::F(FloatKind::F16); + let f128 = K::F(FloatKind::F128); + let good: &[(ConvertKind, K, K)] = &[ + (Trunc, i32, i8), + (Trunc, i64, i32), + (Trunc, K::VI(32), K::VI(16)), + (Zext, K::Bool, i8), + (Zext, K::Bool, i32), + (Zext, i8, i32), + (Zext, i32, i64), + (Zext, K::VI(8), K::VI(32)), + (Sext, i8, i16), + (Sext, i32, i64), + (Sext, K::VI(16), K::VI(32)), + (SiToFp, i32, F32), + (SiToFp, i64, F64), + (SiToFp, K::VI(32), K::VF(FloatKind::F32)), + (UiToFp, i8, F32), + (UiToFp, i64, f128), + (FpToSi, F32, i32), + (FpToSi, F64, i8), + (FpToSi, K::VF(FloatKind::F32), K::VI(32)), + (FpToUi, F32, i32), + (FpToUi, f128, i64), + (FpResize, F32, F64), + (FpResize, F64, F32), + (FpResize, f16, f128), + (FpResize, K::VF(FloatKind::F32), K::VF(FloatKind::F64)), + ]; + for &(kind, src, dst) in good { + assert_eq!( + convert_bad(kind, src, dst), + 0, + "{kind:?} {src:?} -> {dst:?}" + ); + } + let bad: &[(ConvertKind, K, K)] = &[ + // Trunc must narrow, and between ints. + (Trunc, i8, i32), + (Trunc, i32, i32), + (Trunc, F64, F32), + (Trunc, i32, K::Bool), + // Zext / Sext must widen; Sext does not take bool. + (Zext, i32, i8), + (Zext, i32, i32), + (Zext, F32, i64), + (Sext, i32, i8), + (Sext, i32, i32), + (Sext, K::Bool, i32), + (Sext, F32, F64), + // Int -> float and float -> int only. + (SiToFp, F32, F64), + (SiToFp, K::Bool, F32), + (UiToFp, F32, i32), + (FpToSi, i32, F32), + (FpToUi, i32, i64), + (FpToUi, F32, F64), + // FpResize needs two different float types. + (FpResize, F32, F32), + (FpResize, i32, F32), + (FpResize, F32, i32), + // Lane counts must agree. + (Zext, i8, K::VI(32)), + (SiToFp, K::VI(32), F32), + (FpResize, F32, K::VF(FloatKind::F64)), + ]; + for &(kind, src, dst) in bad { + assert_eq!( + convert_bad(kind, src, dst), + 1, + "{kind:?} {src:?} -> {dst:?}" + ); + } + } + + #[test] + fn pointers_are_never_converted_pointer_integer_casts_are_reinterprets() { + for kind in [ + ConvertKind::Trunc, + ConvertKind::Zext, + ConvertKind::Sext, + ConvertKind::SiToFp, + ConvertKind::UiToFp, + ConvertKind::FpToSi, + ConvertKind::FpToUi, + ConvertKind::FpResize, + ] { + assert_eq!(convert_bad(kind, K::Ptr, K::I(64)), 1, "{kind:?}"); + assert_eq!(convert_bad(kind, K::I(32), K::Ptr), 1, "{kind:?}"); + } + } + + #[test] + fn a_convert_without_a_result_is_rejected() { + let mut func = Function::new(); + let i32_t = func.types.intern(int(32)); + let entry = func.append_block(); + let a = func.append_block_param(entry, i32_t); + func.append_stmt_raw( + entry, + Opcode::Convert(crate::function::Convert { + value: a, + kind: ConvertKind::Zext, + }), + ); + func.set_terminator(entry, Terminator::Ret(Ret::none())); + assert_eq!(invalid_operations(&func), 1); + } +} diff --git a/src/isa/aarch64/Cargo.toml b/src/isa/aarch64/Cargo.toml new file mode 100644 index 0000000..74e984b --- /dev/null +++ b/src/isa/aarch64/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "volt-isa-aarch64" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +path = "src/lib.rs" + +[lints] +workspace = true diff --git a/src/emulator/src/aarch64/decode.rs b/src/isa/aarch64/src/decode.rs similarity index 59% rename from src/emulator/src/aarch64/decode.rs rename to src/isa/aarch64/src/decode.rs index 8d10f55..1bff71a 100644 --- a/src/emulator/src/aarch64/decode.rs +++ b/src/isa/aarch64/src/decode.rs @@ -1,4 +1,5 @@ -use super::simd_struct::{Shape, StructDesc}; +use crate::simd_struct::{Shape, StructDesc}; +use crate::{fp, gxf, pauth, simd}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DecodeError { @@ -124,15 +125,61 @@ pub struct Arithmetic { pub width: Width, pub rn: u8, pub rd: u8, - pub immediate: u32, + /// The 12-bit immediate field as encoded, before `shifted` applies. + pub immediate: u16, + /// `lsl #12` on the immediate: `add x0, x1, #0, lsl #12` differs from `add x0, x1, #0`. + pub shifted: bool, +} +impl Arithmetic { + /// The operand value: the immediate field, shifted left by 12 when `shifted`. + pub fn operand(self) -> u32 { + u32::from(self.immediate) << if self.shifted { 12 } else { 0 } + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Addressing { + /// `[xn, #imm]` with the scaled unsigned 12-bit offset (`ldr`, `str`, `prfm`) or, for + /// pairs, the scaled signed 7-bit one (`ldp`, `stp`, `ldnp`, `stnp`). Offset(i64), - Register { rm: u8, extend: Extend, amount: u8 }, + /// `[xn, #simm]` with the unscaled signed 9-bit offset: `ldur`, `stur`, `prfum`, + /// `ldapur`, `stlur`. The offset may be representable as `Offset` too. + Unscaled(i64), + /// `[xn, #simm]` of `ldtr`/`sttr`: the unscaled signed 9-bit offset, but the + /// access is checked as an unprivileged (EL0) one. + Unprivileged(i64), + /// `amount` is `None` when the S bit is clear and `Some(n)` when the offset register + /// is shifted by `n`: for a byte access `Some(0)` is spelled `lsl #0`. + Register { + rm: u8, + extend: Extend, + amount: Option, + }, PreIndex(i64), PostIndex(i64), } +/// The acquire/release class of an `Offset(0)` or `Unscaled` access that is neither a +/// plain load or store nor an exclusive. `ldar`, `stlr` and `ldapr` are the ARMv8 +/// RCsc/RCpc pair; the `L` forms are the LORegion limited-ordering ones. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Ordered { + /// `ldar`: load-acquire. + Acquire, + /// `ldapr`, `ldapur`: load-acquire with only RCpc (processor consistent) ordering. + AcquirePc, + /// `ldlar`: limited-ordering load-acquire. + LimitedAcquire, + /// `stlr`, `stlur`: store-release. + Release, + /// `stllr`: limited-ordering store-release. + LimitedRelease, +} +/// The acquire (`a`) and release (`l`) bits of an exclusive or LSE atomic, spelled +/// `ldaxr`, `stlxr`, `casal`, `ldaddl`, ... +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MemoryOrder { + pub acquire: bool, + pub release: bool, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Memory { pub op: MemoryOp, @@ -141,6 +188,9 @@ pub struct Memory { pub rt: u8, pub addressing: Addressing, pub signed: SignExtend, + /// `Some` for `ldar`, `stlr`, `ldapr`, `ldlar`, `stllr` and the unscaled `ldapur`, + /// `stlur`, `ldapurs*`; the machines run them as the plain access. + pub ordered: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Pair { @@ -151,6 +201,8 @@ pub struct Pair { pub rt2: u8, pub addressing: Addressing, pub signed: SignExtend, + /// `ldnp`, `stnp`: the non-temporal hint form (always `Addressing::Offset`). + pub non_temporal: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Multiply { @@ -189,7 +241,10 @@ pub struct Literal { pub size: Size, pub rt: u8, pub offset: i64, + /// `ldrsw` (literal) sign-extends its word to 64 bits. + pub signed: SignExtend, } +/// `LDXR`, `LDAXR`, `STXR`, `STLXR` and their byte and halfword forms. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Exclusive { pub op: MemoryOp, @@ -197,10 +252,13 @@ pub struct Exclusive { pub rn: u8, pub rt: u8, pub rs: u8, + /// `ldaxr` acquires, `stlxr` releases. The machines ignore it. + pub order: MemoryOrder, } -/// Operation of an LSE atomic. Acquire and release ordering are not modelled -/// separately: each host read-modify-write is sequentially consistent. +/// Operation of an LSE atomic or exclusive pair. The acquire and release bits are kept +/// in `Atomic::order`; the machines do not model them: each host read-modify-write is +/// sequentially consistent. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[repr(u8)] pub enum AtomicKind { @@ -232,6 +290,8 @@ pub struct Atomic { pub rt: u8, /// The second register of `LoadPair` and `StorePair`. pub rt2: u8, + /// The `a` and `l` suffixes of `casal`, `ldaddal`, `swpa`, `ldaxp`, `stlxp`, ... + pub order: MemoryOrder, } impl AtomicKind { /// Number of adjacent words the operation reads and writes. @@ -307,11 +367,43 @@ pub struct Address { pub rd: u8, pub offset: i64, } +/// The operands of a `SYS`-space system instruction (`DC`, `IC`, `TLBI`, `AT`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SysOp { + pub op1: u8, + pub crn: u8, + pub crm: u8, + pub op2: u8, + pub rt: u8, +} +/// `PRFM`/`PRFUM` with a register-based address. `op` is the prefetch operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Prefetch { + pub op: u8, + pub rn: u8, + pub addressing: Addressing, +} +/// `RPRFM , Xm, [Xn]`: a range prefetch. It is a hint and runs as a no-op. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RangePrefetch { + /// The six-bit range prefetch operation. + pub op: u8, + pub rm: u8, + pub rn: u8, +} +/// `PRFM` (literal): a prefetch from a PC-relative address. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PrefetchLiteral { + pub op: u8, + pub offset: i64, +} /// How a modified-immediate instruction combines its immediate with `rd`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ImmCombine { - /// `movi`, `mvni`, `fmov`: replace the register. + /// `movi`, `fmov`: replace the register. Set, + /// `mvni`: replace the register with the (already inverted) pattern. + SetInverted, /// `orr`: `rd |= imm`. Or, /// `bic`: `rd &= !imm`. @@ -320,13 +412,39 @@ pub enum ImmCombine { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdImmediate { pub rd: u8, + /// The encoded 8-bit immediate (`abc:defgh`). + pub imm8: u8, + /// The lane layout and shift the encoding (`cmode`, `op`) selects. + pub form: ImmForm, /// The expanded immediate: the low and high 64 bits (the high half is zero for /// a 64-bit `q == false` form, which also clears the register's upper half). + /// Inverted for `mvni`. pub low: u64, pub high: u64, pub combine: ImmCombine, pub q: bool, } +/// The lane layout of a modified immediate (`AdvSIMDExpandImm` `cmode` classes). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ImmForm { + /// `.8b`/`.16b`: `imm8` in every byte. + Byte, + /// `.4h`/`.8h`: `imm8 << shift` with `shift` 0 or 8 (`lsl`). + Half { shift: u8 }, + /// `.2s`/`.4s`: `imm8 << shift` with `shift` 0, 8, 16 or 24 (`lsl`). + Word { shift: u8 }, + /// `.2s`/`.4s`: `imm8 << shift` with the vacated low bits set, `shift` 8 or 16 + /// (`msl`). + WordMask { shift: u8 }, + /// `movi d`/`.2d`: each bit of `imm8` stands for a 0x00 or 0xff byte. + Double, + /// `fmov .4h`/`.8h`. + FmovHalf, + /// `fmov .2s`/`.4s`. + FmovSingle, + /// `fmov .2d`. + FmovDouble, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdPair { pub op: MemoryOp, @@ -335,6 +453,18 @@ pub struct SimdPair { pub rt: u8, pub rt2: u8, pub addressing: Addressing, + /// `ldnp`, `stnp`: the non-temporal hint form (always `Addressing::Offset`). + pub non_temporal: bool, +} +/// How a [`SimdMemory`] is spelled. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SimdAccess { + /// `ldr`/`str` (`ldur`/`stur` with an unscaled offset) of a `b`, `h`, `s`, `d` or `q` + /// register. + Register, + /// One-register `ld1`/`st1` of a whole `.8b`..`.2d` vector whose elements are `esize` + /// bytes. It moves the same bytes as the `d`/`q` register form. + Ld1 { esize: u8 }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SimdMemory { @@ -343,6 +473,7 @@ pub struct SimdMemory { pub rn: u8, pub rt: u8, pub addressing: Addressing, + pub access: SimdAccess, } /// How a SIMD structure access updates its base register afterwards. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -379,7 +510,10 @@ pub struct SimdExt { pub rd: u8, pub rn: u8, pub rm: u8, + /// The byte index into the `rm:rn` pair, below 8 when `q` is clear. pub imm: u8, + /// The 128-bit form (`.16b`); clear for `.8b`. + pub q: bool, } /// `dup` from an element of a vector register: replicate lane `index` of /// `rn` across every lane of the destination. @@ -428,6 +562,9 @@ pub struct SimdFmov { pub to_fp: bool, /// True for 64-bit (`x`/`d`), false for 32-bit (`w`/`s`). pub double: bool, + /// The upper doubleword of the vector (`Vd.D[1]`); only with `double`. A move + /// to it keeps the lower doubleword. + pub upper: bool, } /// Three-registers-same integer vector operation (`U`, `size` and `opcode` /// fields verified against `llvm-mc` encodings). @@ -520,6 +657,19 @@ pub enum SystemRegister { SpEl0, SpEl1, SpEl2, + SctlrEl2, + TcrEl2, + Ttbr0El2, + Ttbr1El2, + MairEl2, + VbarEl2, + ElrEl2, + SpsrEl2, + EsrEl2, + FarEl2, + HcrEl2, + TpidrEl2, + CntvoffEl2, CurrentEl, SpsrEl1, ElrEl1, @@ -596,11 +746,43 @@ pub enum SystemRegister { IccBpr0El1, IccCtlrEl1, IccIgrpen0El1, + /// Group 1 enable and binary point: the group the firmware and Linux use for IRQs. + IccIgrpen1El1, + IccBpr1El1, + /// Group 1 acknowledge and end of interrupt. No group 1 interrupt is delivered by this + /// model, so an acknowledge reads spurious and an end of interrupt has nothing to end. + IccIar1El1, + IccEoir1El1, /// Stored, but the physical timer never fires. CntpCtlEl0, + /// The physical counter (the virtual count, as CNTVOFF_EL2 is zero without EL2). + CntpctEl0, + /// Physical timer compare value and its 32-bit view, sharing the counter. It never fires. + CntpTvalEl0, + CntpCvalEl0, ActlrEl1, Nzcv, - RazWi, + /// Read-as-zero, write-ignored debug register `DBG{B,W}{V,C}R_EL1` (`op0` 2, `op2` 4 to 7: + /// `DBGBVR`, `DBGBCR`, `DBGWVR`, `DBGWCR`). + RazWi { + crm: u8, + op2: u8, + }, + /// Read-as-zero, write-ignored: Apple IMP-DEF status registers (`op0` 3). + AppleZero { + op1: u8, + crm: u8, + op2: u8, + }, + /// Apple IMP-DEF configuration register the guest reads back, stored in + /// `System::apple[slot]`. + Apple(u8), + /// Apple IMP-DEF register of the banks the SPTM monitor configures (`op1` 4 to 6, and the + /// `op1` 0, 1 and 3 registers in `MONITOR_LOW_BANKS`), which + /// the SPTM monitor reads back; stored in `Cpu::apple_bank`. + AppleBank(u16), + /// PAuth key register `APxAKey{Lo,Hi}_EL1`, stored in `System::pac_keys[slot]`. + PacKey(u8), } impl SystemRegister { pub fn read_only(self) -> bool { @@ -643,6 +825,8 @@ pub struct System { pub read: bool, pub register: SystemRegister, pub rt: u8, + /// The `#imm` of `msr , #imm` (`DaifSet`, `DaifClear`, `SpSelSet`, + /// `PanSet`); zero for the register forms. pub immediate: u8, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -690,10 +874,21 @@ pub struct Call { pub target: i64, pub link: bool, } +/// Which register branch an [`Indirect`] is. `RET` is architecturally a `BR` with a +/// return hint, but analysis (and branch prediction) must tell them apart. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum IndirectKind { + /// `BR Xn`. + Jump, + /// `BLR Xn`: also writes the return address to `X30`. + Call, + /// `RET {Xn}`. + Return, +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Indirect { pub rn: u8, - pub link: bool, + pub kind: IndirectKind, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Variable { @@ -798,23 +993,29 @@ pub enum Instruction { SimdLiteral(SimdLiteral), Exclusive(Exclusive), Atomic(Atomic), - Clrex, + Clrex(u8), Adr(Address), System(System), DcZva(u8), AddressTranslate(AddressTranslate), Eret, - CacheOp, - Tlbi, + CacheOp(SysOp), + Tlbi(SysOp), Nop, - Dsb, - Dmb, - Isb, + /// A hint-space `HINT #n` other than `nop` and the named hints (`bti`, `esb`, + /// `csdb`, ...). Executes as `nop`. + Hint(u8), + Prefetch(Prefetch), + PrefetchLiteral(PrefetchLiteral), + RangePrefetch(RangePrefetch), + Dsb(u8), + Dmb(u8), + Isb(u8), Sev, Sevl, Wfe, Yield, - Trap, + Brk(u16), B(i64), BCond(BranchCondition), Csel(Select), @@ -830,10 +1031,6 @@ pub enum Instruction { SimdStruct(SimdStruct), SimdCompareZero(SimdCompareZero), SimdFmov(SimdFmov), - SimdMove { - rd: u8, - rn: u8, - }, SimdDup { rd: u8, rn: u8, @@ -841,70 +1038,24 @@ pub enum Instruction { q: bool, }, SimdDupElement(SimdDupElement), - /// An instruction the host executes from its raw word (see `host`). - Host(u32), SimdInsert(SimdInsert), SimdInsertElement(SimdInsertElement), SimdExtract(SimdExtract), SimdExt(SimdExt), SimdAlu(SimdAlu), SimdTable(SimdTable), - Svc, + Svc(u16), Psci, Wfi, + /// Scalar floating point; see [`fp`]. + Fp(fp::Op), + /// Integer Advanced SIMD instructions without a structured form; see [`simd`]. + Simd(simd::Op), + /// Pointer authentication; see [`pauth`]. + Pauth(pauth::Op), + /// Guarded execution; see [`gxf`]. + Gxf(gxf::Op), } -impl Instruction { - /// Whether this instruction ends a translated block. With `inline_memory`, - /// plain loads and stores are compiled with an inline fast path and no - /// longer end the block (their slow path exits mid-block instead). - pub fn terminates_with(&self, inline_memory: bool) -> bool { - if inline_memory - && matches!( - self, - Instruction::Memory(_) - | Instruction::Pair(_) - | Instruction::Literal(_) - | Instruction::SimdMemory(_) - | Instruction::SimdPair(_) - ) - { - return false; - } - self.terminates() - } - pub fn terminates(&self) -> bool { - use Instruction::*; - matches!( - self, - Memory(_) - | Exclusive(_) - | Atomic(_) - | Literal(_) - | SimdLiteral(_) - | Pair(_) - | SimdPair(_) - | SimdMemory(_) - | SimdStruct(_) - | B(_) - | BCond(_) - | TestBranch(_) - | Call(_) - | Indirect(_) - | Svc - | Psci - | Wfi - | Wfe - | Trap - | Isb - | DcZva(_) - | AddressTranslate(_) - | Host(_) - | Eret - | Tlbi - ) - } -} - /// Whether `word` is architecturally UNDEFINED on this CPU, as opposed to an /// instruction the emulator does not implement. Execution takes an Undefined /// Instruction exception (XNU's `TRAP_DEBUGGER` word `0xe7ffdeff` relies on it). These @@ -970,6 +1121,45 @@ fn extension(n: u32) -> Extend { Extend::Sxtx, ][(n & 7) as usize] } +/// The `[xn, rm{, ext{ #amount}}]` address of a register-offset load, store or prefetch. +/// `scale` is log2 of the access size, the amount applied when the S bit is set. Options +/// 0, 1, 4 and 5 (the byte and halfword extensions) and a non-zero bits 11:10 are +/// unallocated. +fn register_offset(word: u32, scale: u32) -> Result { + let option = (word >> 13) & 7; + if option & 2 == 0 || word & 0x0c00 != 0x0800 { + return Err(DecodeError::UnsupportedInstruction); + } + Ok(Addressing::Register { + rm: reg(word, 16), + extend: extension(option), + amount: (word & 0x1000 != 0).then_some(scale as u8), + }) +} +/// The address of a load/store-register word without the unsigned-offset or register +/// form: bits 11:10 select unscaled, post-index, unprivileged or pre-index with the +/// signed 9-bit offset. The SIMD&FP forms have no unprivileged variant. +fn immediate_offset(word: u32, vector: bool) -> Result { + let offset = sign_extend(word >> 12, 9); + match (word >> 10) & 3 { + 0 => Ok(Addressing::Unscaled(offset)), + 1 => Ok(Addressing::PostIndex(offset)), + 2 if !vector => Ok(Addressing::Unprivileged(offset)), + 3 => Ok(Addressing::PreIndex(offset)), + _ => Err(DecodeError::UnsupportedInstruction), + } +} +/// The sign extension of a general-register load or store: `opc` is bits 23:22 and +/// `size` the access size. `opc` 0 stores and 1 loads zero-extended; 2 is `ldrs[bhw] Xt` +/// and 3 is `ldrs[bh] Wt`. +fn load_extension(opc: u32, size: Size) -> Result { + match opc { + 0 | 1 => Ok(SignExtend::None), + 2 if size != Size::Double => Ok(SignExtend::To64), + 3 if size == Size::Byte || size == Size::Half => Ok(SignExtend::To32), + _ => Err(DecodeError::UnsupportedInstruction), + } +} fn sign_extend(n: u32, bits: u32) -> i64 { ((n << (32 - bits)) as i32 >> (32 - bits)) as i64 } @@ -1015,9 +1205,8 @@ fn bitmask(word: u32, width: Width) -> Result { Ok(immediate) } /// Decode an Advanced SIMD structure load/store word (`word & 0xbe00_0000 == -/// 0x0c00_0000`) into its descriptor and post-index, or `None` for the -/// whole-register LD1/ST1 forms that `SimdMemory` handles. -fn simd_struct(word: u32) -> Result, DecodeError> { +/// 0x0c00_0000`) into its descriptor and post-index. +fn simd_struct(word: u32) -> Result<(StructDesc, StructPost), DecodeError> { use DecodeError::UnsupportedInstruction as Bad; let q = word & 0x4000_0000 != 0; let rm = reg(word, 16); @@ -1058,9 +1247,6 @@ fn simd_struct(word: u32) -> Result, DecodeErro if desc.interleaved && !q && size == 3 { return Err(Bad); } - if desc.registers == 1 && !matches!(post, StructPost::Register(_)) { - return Ok(None); - } } else { // Bits 15:13 give the element scale (and, with R, the structure count); // the lane index is spread over Q, S and size. @@ -1079,8 +1265,127 @@ fn simd_struct(word: u32) -> Result, DecodeErro }; (desc.shape, desc.esize, desc.lane) = (shape, esize, lane); } - Ok(Some((desc, post))) + Ok((desc, post)) +} +/// Number of Apple IMP-DEF configuration registers `System::apple` holds. +pub const APPLE_SLOTS: usize = 21; +/// Number of `S3__c15` registers `Cpu::apple_bank` holds: eight banks of +/// `crm` x `op2` (16 x 8). +pub const APPLE_BANK_SLOTS: usize = 8 * 128; + +/// The `Cpu::apple_bank` slot of `S3__c15_c_`. +pub const fn apple_bank_slot(op1: u32, crm: u32, op2: u32) -> usize { + ((op1 as usize) << 7) | ((crm as usize) << 3) | op2 as usize +} + +/// The `S3_{0,1,3}_c15` registers (`op1`, `crm`, `op2`) the SPTM monitor's code accesses +/// that the kernel-side table above does not name, found by scanning its `mrs`/`msr` +/// words. Their roles are not identified; they are stored so the monitor reads back what +/// it wrote. Other registers in those banks stay unsupported. +const MONITOR_LOW_BANKS: [(u32, u32, u32); 25] = [ + (0, 0, 0), + (0, 1, 0), + (0, 1, 2), + (0, 1, 3), + (0, 2, 0), + (0, 2, 1), + (0, 3, 0), + (0, 4, 0), + (0, 4, 1), + (0, 5, 0), + (0, 6, 0), + (0, 7, 0), + (0, 9, 0), + (0, 9, 1), + (0, 10, 1), + (0, 11, 0), + (0, 11, 2), + (0, 14, 0), + (0, 15, 0), + (0, 15, 2), + (0, 15, 5), + (1, 8, 2), + (3, 8, 0), + (3, 9, 0), + (3, 10, 0), +]; + +/// The `S3_*` implementation-defined registers the M1 kernel and PMU driver touch +/// at EL1 (`arch/arm64/include/asm/apple_m1_pmu.h`, `irq-apple-aic.c`). Configuration +/// is stored so the guest reads back what it wrote. Status registers (IPI pending, +/// PMU interrupt active, PMU state) read as zero: this model raises no fast IPIs and +/// no PMU interrupts. Anything else falls through to the unsupported path. +fn apple_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option { + use SystemRegister::*; + if op0 != 3 || crn != 15 { + return None; + } + let zero = AppleZero { + op1: op1 as u8, + crm: crm as u8, + op2: op2 as u8, + }; + Some(match (op1, crm, op2) { + (1, 0, 0) => Apple(0), + (1, 1, 0) => Apple(1), + (1, 2, 0) => Apple(2), + (1, 3, 0) => Apple(3), + (1, 4, 0) => Apple(4), + (1, 5, 0) => Apple(5), + (1, 6, 0) => Apple(6), + (1, 13, 0) => zero, + (2, 0, 0) => Apple(7), + (2, 1, 0) => Apple(8), + (2, 2, 0) => Apple(9), + (2, 3, 0) => Apple(10), + (2, 4, 0) => Apple(11), + (2, 5, 0) => Apple(12), + (2, 6, 0) => Apple(13), + (2, 7, 0) => Apple(14), + (2, 9, 0) => Apple(15), + (2, 10, 0) => Apple(16), + (5, 0, 0) => Apple(17), + (5, 0, 1) => Apple(18), + (5, 1, 1) => zero, + (5, 3, 1) => Apple(19), + (7, 0, 4) => Apple(20), + (7, 6, 4) => zero, + // The S3_5_c15 state registers the kernel reads during early boot. None are named + // in the XNU source tree that is available here, so they read as zero and writes + // are dropped until their roles are identified. + (5, 2..=7, 0) => zero, + // The EL2 and guarded-level banks the SPTM monitor configures: stored, so a + // read-modify-write or a read-back sees what was written. + (4..=6, _, _) => AppleBank(apple_bank_slot(op1, crm, op2) as u16), + _ if MONITOR_LOW_BANKS.contains(&(op1, crm, op2)) => { + AppleBank(apple_bank_slot(op1, crm, op2) as u16) + } + _ => return None, + }) +} + +/// The PAuth key registers `S3_0_C2_C{1,2,3}_*`: APIA, APIB, APDA, APDB and APGA, low +/// word first (`System::pac_keys`). +fn pac_key_register(op0: u32, op1: u32, crn: u32, crm: u32, op2: u32) -> Option { + if op0 != 3 || op1 != 0 || crn != 2 { + return None; + } + let slot = match (crm, op2) { + (1, 0) => 0, + (1, 1) => 1, + (1, 2) => 2, + (1, 3) => 3, + (2, 0) => 4, + (2, 1) => 5, + (2, 2) => 6, + (2, 3) => 7, + (3, 0) => 8, + (3, 1) => 9, + _ => return None, + }; + Some(SystemRegister::PacKey(slot)) } + fn system_register( op0: u32, op1: u32, @@ -1090,7 +1395,16 @@ fn system_register( ) -> Result { use SystemRegister::*; if op0 == 2 && op1 == 0 && crn == 0 && (4..=7).contains(&op2) { - return Ok(RazWi); + return Ok(RazWi { + crm: crm as u8, + op2: op2 as u8, + }); + } + if let Some(register) = apple_register(op0, op1, crn, crm, op2) { + return Ok(register); + } + if let Some(register) = pac_key_register(op0, op1, crn, crm, op2) { + return Ok(register); } if op1 == 0 && crn == 4 && op2 == 5 { return Ok(SpSelSet); @@ -1110,6 +1424,19 @@ fn system_register( 0x0410 => SpEl0, 0x4410 => SpEl1, 0x6410 => SpEl2, + 0x4100 => SctlrEl2, + 0x4202 => TcrEl2, + 0x4200 => Ttbr0El2, + 0x4201 => Ttbr1El2, + 0x4a20 => MairEl2, + 0x4c00 => VbarEl2, + 0x4401 => ElrEl2, + 0x4400 => SpsrEl2, + 0x4520 => EsrEl2, + 0x4600 => FarEl2, + 0x4110 => HcrEl2, + 0x4d02 => TpidrEl2, + 0x4e03 => CntvoffEl2, 0x0422 => CurrentEl, 0x0400 => SpsrEl1, 0x0401 => ElrEl1, @@ -1147,7 +1474,14 @@ fn system_register( 0x0c83 => IccBpr0El1, 0x0cc4 => IccCtlrEl1, 0x0cc6 => IccIgrpen0El1, + 0x0cc7 => IccIgrpen1El1, + 0x0cc3 => IccBpr1El1, 0x3e21 => CntpCtlEl0, + 0x3e01 => CntpctEl0, + 0x3e20 => CntpTvalEl0, + 0x3e22 => CntpCvalEl0, + 0x0cc0 => IccIar1El1, + 0x0cc1 => IccEoir1El1, 0x0101 => ActlrEl1, 0x0000 => MidrEl1, 0x0005 => MpidrEl1, @@ -1181,7 +1515,49 @@ fn system_register( }) } +/// The 16-bit pattern of the 8-bit floating-point immediate `imm8` (`VFPExpandImm` +/// for half precision), as the vector `fmov` takes it. +pub fn expand_half_immediate(imm8: u64) -> u64 { + let sign = imm8 >> 7; + let b = (imm8 >> 6) & 1; + (sign << 15) | ((b ^ 1) << 14) | (b * 0b11 << 12) | ((imm8 & 0x3f) << 6) +} + +/// The 32- or 64-bit pattern of the 8-bit floating-point immediate `imm8` +/// (`VFPExpandImm`), as `fmov` and the vector `fmov` take it. +pub fn expand_immediate(imm8: u64, double: bool) -> u64 { + let sign = imm8 >> 7; + let b = (imm8 >> 6) & 1; + let fraction = imm8 & 0x3f; + if double { + (sign << 63) | ((b ^ 1) << 62) | (b * 0xff << 54) | (fraction << 48) + } else { + (sign << 31) | ((b ^ 1) << 30) | (b * 0x1f << 25) | (fraction << 19) + } +} + +/// Decode one instruction word. +/// +/// The families in [`fp`], [`simd`], [`pauth`] and [`gxf`] are tried first and take +/// precedence over the structured forms of this module wherever both recognise a word: +/// the emulator executes those families from their decoded form, never by lifting them. pub fn decode(word: u32) -> Result { + if let Some(op) = fp::decode(word) { + return Ok(Instruction::Fp(op)); + } + if let Some(op) = simd::decode(word) { + return Ok(Instruction::Simd(op)); + } + if let Some(op) = pauth::decode(word) { + return Ok(Instruction::Pauth(op)); + } + if let Some(op) = gxf::decode(word) { + return Ok(Instruction::Gxf(op)); + } + structured(word) +} + +fn structured(word: u32) -> Result { use DecodeError::UnsupportedInstruction; use Instruction::*; let width = width(word); @@ -1346,9 +1722,20 @@ pub fn decode(word: u32) -> Result { if word & 0x001f_0000 != 0x001f_0000 || word & 0x0000_fc1f != 0 { return Err(UnsupportedInstruction); } - return match (word >> 21) & 7 { - 0 | 2 => Ok(Indirect(self::Indirect { rn, link: false })), - 1 => Ok(Indirect(self::Indirect { rn, link: true })), + // opc is bits 24:21; bit 24 set is the authenticated forms (`braa`, `retaa`, ...). + return match (word >> 21) & 15 { + 0 => Ok(Indirect(self::Indirect { + rn, + kind: IndirectKind::Jump, + })), + 1 => Ok(Indirect(self::Indirect { + rn, + kind: IndirectKind::Call, + })), + 2 => Ok(Indirect(self::Indirect { + rn, + kind: IndirectKind::Return, + })), 4 if rn == 31 => Ok(Eret), _ => Err(UnsupportedInstruction), }; @@ -1363,26 +1750,67 @@ pub fn decode(word: u32) -> Result { width, rn, rd, - immediate: ((word >> 10) & 0xfff) << if word & 0x0040_0000 != 0 { 12 } else { 0 }, + immediate: ((word >> 10) & 0xfff) as u16, + shifted: word & 0x0040_0000 != 0, })); } - if word & 0xffc0_0000 == 0xf980_0000 - || word & 0xffe0_0c00 == 0xf8a0_0800 - || word & 0xffe0_0c00 == 0xf880_0000 - || word & 0xff00_0000 == 0xd800_0000 - { - return Ok(Nop); + // PRFM (unsigned offset), PRFUM (unscaled offset) and PRFM (register). They + // keep their operand and execute as `nop`. + if word & 0xffc0_0000 == 0xf980_0000 { + return Ok(Prefetch(self::Prefetch { + op: rd, + rn, + addressing: Addressing::Offset((((word >> 10) & 0xfff) << 3) as i64), + })); + } + if word & 0xffe0_0c00 == 0xf880_0000 { + return Ok(Prefetch(self::Prefetch { + op: rd, + rn, + addressing: Addressing::Unscaled(sign_extend(word >> 12, 9)), + })); + } + if word & 0xffe0_0c00 == 0xf8a0_0800 { + let addressing = register_offset(word, 3)?; + // RPRFM takes the PRFM (register) slots whose Rt is 0b11xxx; its 6-bit operation + // is `option<2>:option<0>:S:Rt<2:0>`. + if rd >= 24 { + let option = (word >> 13) & 7; + return Ok(RangePrefetch(self::RangePrefetch { + op: ((option >> 2) << 5 + | (option & 1) << 4 + | ((word >> 12) & 1) << 3 + | u32::from(rd & 7)) as u8, + rm, + rn, + })); + } + return Ok(Prefetch(self::Prefetch { + op: rd, + rn, + addressing, + })); + } + // PRFM (literal). + if word & 0xff00_0000 == 0xd800_0000 { + return Ok(PrefetchLiteral(self::PrefetchLiteral { + op: rd, + offset: sign_extend(word >> 5, 19) << 2, + })); } if word & 0x3f00_0000 == 0x1800_0000 { - let size = match word >> 30 { - 0 => Size::Word, - 1 => Size::Double, + let (size, signed) = match word >> 30 { + 0 => (Size::Word, SignExtend::None), + 1 => (Size::Double, SignExtend::None), + // LDRSW (literal). + 2 => (Size::Word, SignExtend::To64), _ => return Err(UnsupportedInstruction), }; return Ok(Literal(self::Literal { size, rt: rd, offset: sign_extend(word >> 5, 19) << 2, + signed, })); } // LDR (literal, SIMD&FP): S, D or Q register from a PC-relative address. @@ -1400,9 +1828,14 @@ pub fn decode(word: u32) -> Result { })); } // LSE atomics (ARMv8.1): LDADD/LDCLR/LDEOR/LDSET/LDSMAX/LDSMIN/LDUMAX/LDUMIN and - // SWP share one encoding, split by o3 (bit 15) and opc (bits 14:12). LDAPR, the - // RCpc load, is the o3=1, opc=100 slot with Rs unused. + // SWP share one encoding, split by o3 (bit 15) and opc (bits 14:12); A (bit 23) is + // acquire and R (bit 22) release. LDAPR, the RCpc load, is the A=1, R=0, o3=1, + // opc=100 slot with Rs unused. if word & 0x3f20_0c00 == 0x3820_0000 { + let order = MemoryOrder { + acquire: word & 0x0080_0000 != 0, + release: word & 0x0040_0000 != 0, + }; let kind = match (word & 0x8000 != 0, (word >> 12) & 7) { (false, 0) => AtomicKind::Add, (false, 1) => AtomicKind::Clr, @@ -1413,7 +1846,7 @@ pub fn decode(word: u32) -> Result { (false, 6) => AtomicKind::UMax, (false, 7) => AtomicKind::UMin, (true, 0) => AtomicKind::Swap, - (true, 4) if rm == 31 => { + (true, 4) if rm == 31 && order.acquire && !order.release => { return Ok(Memory(self::Memory { op: MemoryOp::Load, size: size(word), @@ -1421,6 +1854,7 @@ pub fn decode(word: u32) -> Result { rt: rd, addressing: Addressing::Offset(0), signed: SignExtend::None, + ordered: Some(Ordered::AcquirePc), })); } _ => return Err(UnsupportedInstruction), @@ -1432,9 +1866,15 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rt2: 0, + order, })); } // CAS/CASP (ARMv8.1): compare with Rs (the pair Rs, Rs+1 for CASP); store Rt on a match. + // L (bit 22) is acquire and o0 (bit 15) release. + let cas_order = MemoryOrder { + acquire: word & 0x0040_0000 != 0, + release: word & 0x8000 != 0, + }; if word & 0x3fa0_7c00 == 0x08a0_7c00 { return Ok(Atomic(self::Atomic { kind: AtomicKind::Cas, @@ -1443,6 +1883,7 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rt2: 0, + order: cas_order, })); } if word & 0xbfa0_7c00 == 0x0820_7c00 { @@ -1462,13 +1903,20 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rt2: 0, + order: cas_order, })); } if word & 0x3f00_0000 == 0x0800_0000 { + // o0 (bit 15): release for a store, acquire for a load. + let load = word & 0x0040_0000 != 0; + let o0 = word & 0x8000 != 0; + let order = MemoryOrder { + acquire: load && o0, + release: !load && o0, + }; if word & 0x0020_0000 != 0 { // LDXP/LDAXP/STXP/STLXP: a pair of words or doublewords; the load form has // no status register. - let load = word & 0x0040_0000 != 0; if word & 0x8000_0000 == 0 || word & 0x0080_0000 != 0 || (load && rm != 31) { return Err(UnsupportedInstruction); } @@ -1487,10 +1935,10 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rt2: reg(word, 10), + order, })); } let size = size(word); - let load = word & 0x0040_0000 != 0; let op = if load { MemoryOp::Load } else { @@ -1501,7 +1949,8 @@ pub fn decode(word: u32) -> Result { return Err(UnsupportedInstruction); } if word & 0x0080_0000 != 0 { - if rs != 31 || (load && word & 0x8000 == 0) { + // LDAR/STLR (o0 set) and the LORegion LDLAR/STLLR (o0 clear). + if rs != 31 { return Err(UnsupportedInstruction); } return Ok(Memory(self::Memory { @@ -1511,6 +1960,12 @@ pub fn decode(word: u32) -> Result { rt: rd, addressing: Addressing::Offset(0), signed: SignExtend::None, + ordered: Some(match (load, o0) { + (true, true) => Ordered::Acquire, + (true, false) => Ordered::LimitedAcquire, + (false, true) => Ordered::Release, + (false, false) => Ordered::LimitedRelease, + }), })); } if load && rs != 31 { @@ -1522,46 +1977,59 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, rs, + order, })); } - // LD1..LD4 / ST1..ST4 (multiple and single structures) and LD1R..LD4R. - // A single-register LD1/ST1 without a register post-index is left to the - // `SimdMemory` blocks below, which the JIT inlines. - if word & 0xbe00_0000 == 0x0c00_0000 - && let Some((desc, post)) = simd_struct(word)? - { + // LD1..LD4 / ST1..ST4 (multiple and single structures) and LD1R..LD4R. A single + // register without a register post-index moves a whole vector like `ldr d`/`ldr q`, + // which the JIT inlines: that form is a `SimdMemory` that remembers its arrangement. + if word & 0xbe00_0000 == 0x0c00_0000 { + let (desc, post) = simd_struct(word)?; + if desc.shape == Shape::Multiple + && desc.registers == 1 + && !matches!(post, StructPost::Register(_)) + { + let bytes = desc.vector_bytes(); + return Ok(SimdMemory(self::SimdMemory { + op: if desc.store { + MemoryOp::Store + } else { + MemoryOp::Load + }, + bytes, + rn, + rt: desc.rt, + addressing: if post == StructPost::Immediate { + Addressing::PostIndex(i64::from(bytes)) + } else { + Addressing::Offset(0) + }, + access: SimdAccess::Ld1 { esize: desc.esize }, + })); + } return Ok(SimdStruct(self::SimdStruct { desc, rn, post })); } - // LD1/ST1 (one structure): a full or half-width SIMD register. - // - // bits 29:24 are `001100`, bits 14:12 are `111`, bit 23 selects the - // no-offset vs post-index addressing; bits 21:16 are the `size:opc` - // structure field, which must be 0b000000 or 0b011111. - if word & 0xbf00_8000 == 0x0c00_0000 && word & 0x0000_7000 == 0x0000_7000 { - let bytes = if word & 0x4000_0000 != 0 { 16 } else { 8 }; - let post_index = word & 0x0080_0000 != 0; - let index = (word >> 16) & 0x3f; - let addressing = if post_index { - if index != 31 { - return Err(UnsupportedInstruction); - } - Addressing::PostIndex(bytes as i64) - } else { - if index != 0 { - return Err(UnsupportedInstruction); - } - Addressing::Offset(0) - }; - return Ok(SimdMemory(self::SimdMemory { - op: if word & 0x0040_0000 != 0 { - MemoryOp::Load - } else { + // LDAPUR/STLUR and LDAPURS* (FEAT_LRCPC2): the RCpc/release accesses with an unscaled + // offset. + if word & 0x3f20_0c00 == 0x1900_0000 { + let opc = (word >> 22) & 3; + let size = size(word); + return Ok(Memory(self::Memory { + op: if opc == 0 { MemoryOp::Store + } else { + MemoryOp::Load }, - bytes: bytes as u8, + size, rn, rt: rd, - addressing, + addressing: Addressing::Unscaled(sign_extend(word >> 12, 9)), + signed: load_extension(opc, size)?, + ordered: Some(if opc == 0 { + Ordered::Release + } else { + Ordered::AcquirePc + }), })); } if word & 0x3a00_0000 == 0x3800_0000 { @@ -1585,33 +2053,9 @@ pub fn decode(word: u32) -> Result { let addressing = if word & 0x0100_0000 != 0 { Addressing::Offset((((word >> 10) & 0xfff) * bytes as u32) as i64) } else if word & 0x0020_0000 != 0 { - if word & 0x0000_0c00 != 0x800 { - return Err(UnsupportedInstruction); - } - let extend = match (word >> 13) & 7 { - 2 => Extend::Uxtw, - 3 => Extend::Uxtx, - 6 => Extend::Sxtw, - 7 => Extend::Sxtx, - _ => return Err(UnsupportedInstruction), - }; - Addressing::Register { - rm, - extend, - amount: if word & 0x1000 != 0 { - (bytes as u32).trailing_zeros() as u8 - } else { - 0 - }, - } + register_offset(word, (bytes as u32).trailing_zeros())? } else { - let offset = sign_extend(word >> 12, 9); - match (word >> 10) & 3 { - 0 | 2 => Addressing::Offset(offset), - 1 => Addressing::PostIndex(offset), - 3 => Addressing::PreIndex(offset), - _ => return Err(UnsupportedInstruction), - } + immediate_offset(word, true)? }; return Ok(SimdMemory(self::SimdMemory { op: if is_load { @@ -1623,46 +2067,18 @@ pub fn decode(word: u32) -> Result { rn, rt: rd, addressing, + access: SimdAccess::Register, })); } let opc = (word >> 22) & 3; let size = size(word); - let signed = match opc { - 0 | 1 => SignExtend::None, - 2 if size != Size::Double => SignExtend::To64, - 3 if size == Size::Byte || size == Size::Half => SignExtend::To32, - _ => return Err(UnsupportedInstruction), - }; + let signed = load_extension(opc, size)?; let addressing = if word & 0x0100_0000 != 0 { Addressing::Offset((((word >> 10) & 0xfff) * size as u32) as i64) } else if word & 0x0020_0000 != 0 { - if word & 0x0000_0c00 != 0x800 { - return Err(UnsupportedInstruction); - } - let extend = match (word >> 13) & 7 { - 2 => Extend::Uxtw, - 3 => Extend::Uxtx, - 6 => Extend::Sxtw, - 7 => Extend::Sxtx, - _ => return Err(UnsupportedInstruction), - }; - Addressing::Register { - rm, - extend, - amount: if word & 0x1000 != 0 { - (size as u8).trailing_zeros() as u8 - } else { - 0 - }, - } + register_offset(word, (size as u32).trailing_zeros())? } else { - let offset = sign_extend(word >> 12, 9); - match (word >> 10) & 3 { - 0 | 2 => Addressing::Offset(offset), - 1 => Addressing::PostIndex(offset), - 3 => Addressing::PreIndex(offset), - _ => return Err(UnsupportedInstruction), - } + immediate_offset(word, false)? }; return Ok(Memory(self::Memory { op: if opc == 0 { @@ -1675,24 +2091,28 @@ pub fn decode(word: u32) -> Result { rt: rd, addressing, signed, + ordered: None, })); } if word & 0xffff_f01f == 0xd503_201f { - return Ok(match (word >> 5) & 127 { + let hint = ((word >> 5) & 127) as u8; + return Ok(match hint { + 0 => Nop, 1 => Yield, 2 => Wfe, 3 => Wfi, 4 => Sev, 5 => Sevl, - _ => Nop, + _ => Hint(hint), }); } if word & 0xffff_f01f == 0xd503_301f { + let option = ((word >> 8) & 15) as u8; return match (word >> 5) & 7 { - 2 => Ok(Clrex), - 4 => Ok(Dsb), - 5 => Ok(Dmb), - 6 => Ok(Isb), + 2 => Ok(Clrex(option)), + 4 => Ok(Dsb(option)), + 5 => Ok(Dmb(option)), + 6 => Ok(Isb(option)), _ => Err(UnsupportedInstruction), }; } @@ -1711,11 +2131,18 @@ pub fn decode(word: u32) -> Result { if word & 0xfff8_0000 == 0xd508_0000 { let crn = (word >> 12) & 15; let crm = (word >> 8) & 15; + let sys = SysOp { + op1: ((word >> 16) & 7) as u8, + crn: crn as u8, + crm: crm as u8, + op2: ((word >> 5) & 7) as u8, + rt: rd, + }; if crn == 8 { - return Ok(Tlbi); + return Ok(Tlbi(sys)); } if crn == 7 && matches!(crm, 1 | 5 | 6 | 10 | 11 | 14) { - return Ok(CacheOp); + return Ok(CacheOp(sys)); } return Err(UnsupportedInstruction); } @@ -1740,25 +2167,34 @@ pub fn decode(word: u32) -> Result { | SystemRegister::OslarEl1 | SystemRegister::OsdlrEl1 | SystemRegister::OslsrEl1 - | SystemRegister::RazWi => 2, + | SystemRegister::RazWi { .. } => 2, _ => 3, }; if (word >> 19) & 3 != expected || (read && expected == 0) { return Err(UnsupportedInstruction); } + // `msr , #imm` has no register operand (Rt must be 31), and PAN and + // SPSel take a one-bit immediate; any other word is unallocated. + let immediate = ((word >> 8) & 15) as u8; + let pstate = expected == 0; + if pstate + && (rd != 31 + || (matches!(register, SystemRegister::SpSelSet | SystemRegister::PanSet) + && immediate > 1)) + { + return Err(UnsupportedInstruction); + } return Ok(System(self::System { read, register, rt: rd, - immediate: ((word >> 8) & 15) as u8, + immediate: if pstate { immediate } else { 0 }, })); } - if word & 0xffe0_f000 == 0xd420_0000 { - return if word & 0x0000_ffe0 == 0x20 { - Ok(Trap) - } else { - Err(UnsupportedInstruction) - }; + // BRK #imm16: opc 001 with LL 00. Its immediate is the whole field, so any value + // decodes; the trap carries it to the exception syndrome. + if word & 0xffe0_001f == 0xd420_0000 { + return Ok(Brk((word >> 5) as u16)); } if word & 0x1f00_0000 == 0x1000_0000 { let page = word & 0x8000_0000 != 0; @@ -1819,6 +2255,8 @@ pub fn decode(word: u32) -> Result { let vector = word & 0x0400_0000 != 0; let load = word & 0x0040_0000 != 0; let opc = word >> 30; + // Bits 24:23 are 00 for the no-allocate pair (LDNP, STNP). + let non_temporal = (word >> 23) & 7 == 0; if vector { let bytes = match opc { 0 => 4, // S @@ -1844,11 +2282,12 @@ pub fn decode(word: u32) -> Result { rt: rd, rt2: reg(word, 10), addressing, + non_temporal, })); } let load = word & 0x0040_0000 != 0; let opc = word >> 30; - if opc == 1 && !load { + if opc == 1 && (!load || non_temporal) { return Err(UnsupportedInstruction); } let size = match opc { @@ -1874,6 +2313,7 @@ pub fn decode(word: u32) -> Result { rt: rd, rt2: reg(word, 10), addressing, + non_temporal, signed: if opc == 1 { SignExtend::To64 } else { @@ -1884,38 +2324,64 @@ pub fn decode(word: u32) -> Result { // Advanced SIMD modified immediate: MOVI, MVNI, ORR, BIC and FMOV (vector). // `AdvSIMDExpandImm` selects the lane pattern from `cmode` and `op`. // `immh` (bits 22:19) is zero here; a non-zero `immh` is a shift by immediate. + // Bit 11 (`o2`) is zero except in the half-precision `fmov`. if word & 0x9ff8_0400 == 0x0f00_0400 { let q = word & 0x4000_0000 != 0; let op = word & 0x2000_0000 != 0; + let o2 = word & 0x0000_0800 != 0; let cmode = (word >> 12) & 0xf; - let imm8 = u64::from((((word >> 16) & 7) << 5) | ((word >> 5) & 0x1f)); + let imm8 = (((word >> 16) & 7) << 5) | ((word >> 5) & 0x1f); let rd = (word & 0x1f) as u8; let twice = |v: u64| v | (v << 32); - let imm = match cmode { + let wide = |lane: u64| lane * 0x0001_0001_0001_0001; + let wide_imm = u64::from(imm8); + let (form, pattern) = match cmode { // 32-bit lanes, shifted left by 0, 8, 16 or 24. - 0..=7 => twice(imm8 << (8 * (cmode >> 1))), + 0..=7 if !o2 => { + let shift = 8 * (cmode >> 1); + ( + ImmForm::Word { shift: shift as u8 }, + twice(wide_imm << shift), + ) + } // 16-bit lanes, shifted left by 0 or 8. - 8..=11 => { - let lane = imm8 << (8 * ((cmode >> 1) & 1)); - twice(lane | (lane << 16)) + 8..=11 if !o2 => { + let shift = 8 * ((cmode >> 1) & 1); + ( + ImmForm::Half { shift: shift as u8 }, + wide(wide_imm << shift), + ) } // 32-bit lanes with the vacated low bits set to ones. - 12 => twice((imm8 << 8) | 0xff), - 13 => twice((imm8 << 16) | 0xffff), - // Bytes: imm8 in every byte, or each bit of imm8 as a 0x00/0xff byte. - 14 if !op => imm8 * 0x0101_0101_0101_0101, - 14 => (0..8).fold(0, |acc, bit| { - acc | (if (imm8 >> bit) & 1 != 0 { 0xff } else { 0 }) << (8 * bit) - }), - // FMOV: the 8-bit float immediate widened to f32 (twice) or f64. - _ if !op => twice(super::fp::expand_immediate(imm8, false)), - _ if q => super::fp::expand_immediate(imm8, true), + 12 | 13 if !o2 => { + let shift = 8 + 8 * (cmode & 1); + ( + ImmForm::WordMask { shift: shift as u8 }, + twice((wide_imm << shift) | ((1 << shift) - 1)), + ) + } + // Bytes: imm8 in every byte. + 14 if !o2 && !op => (ImmForm::Byte, wide_imm * 0x0101_0101_0101_0101), + // Each bit of imm8 as a 0x00/0xff byte, in the one 64-bit lane. + 14 if !o2 => ( + ImmForm::Double, + (0..8).fold(0, |acc, bit| { + acc | (if (wide_imm >> bit) & 1 != 0 { 0xff } else { 0 }) << (8 * bit) + }), + ), + // FMOV: the 8-bit float immediate widened to f16, f32 (twice) or f64. + 15 if !op && o2 => (ImmForm::FmovHalf, wide(expand_half_immediate(wide_imm))), + 15 if !op => ( + ImmForm::FmovSingle, + twice(expand_immediate(wide_imm, false)), + ), + 15 if q && !o2 => (ImmForm::FmovDouble, expand_immediate(wide_imm, true)), _ => return Err(UnsupportedInstruction), }; // Odd cmode below 12 is ORR/BIC; MOVI becomes MVNI (inverted) with op set. let (value, combine) = if cmode < 12 && cmode & 1 == 1 { ( - imm, + pattern, if op { ImmCombine::AndNot } else { @@ -1923,16 +2389,18 @@ pub fn decode(word: u32) -> Result { }, ) } else if cmode < 14 && op { - (!imm, ImmCombine::Set) + (!pattern, ImmCombine::SetInverted) } else { - (imm, ImmCombine::Set) + (pattern, ImmCombine::Set) }; return Ok(SimdImm(SimdImmediate { rd, - low: value, - high: if q { value } else { 0 }, + imm8: imm8 as u8, + form, combine, q, + low: value, + high: if q { value } else { 0 }, })); } if word & 0xbf3f_fc00 == 0x0e20_9800 { @@ -1943,8 +2411,9 @@ pub fn decode(word: u32) -> Result { } return Ok(SimdCompareZero(self::SimdCompareZero { rd, rn, size, q })); } - // FMOV between a general register and an S/D register: a pure bit move. - if word & 0x7ebe_fc00 == 0x1e26_0000 { + // FMOV between a general register and an S/D register: a pure bit move. Bit 24 + // must be clear; the floating-point 3-source group (FMADD and friends) has it set. + if word & 0x7fbe_fc00 == 0x1e26_0000 { let double64 = word & 0x8000_0000 != 0; let double_type = word & 0x0040_0000 != 0; if double64 != double_type { @@ -1955,12 +2424,18 @@ pub fn decode(word: u32) -> Result { rn, to_fp: word & 0x0001_0000 != 0, double: double64, + upper: false, })); } - // Scalar floating point and integer SIMD the JIT does not compile: `host::run` - // executes the word. - if super::host::is_supported(word) { - return Ok(Host(word)); + // `fmov Vd.D[1], Xn` / `fmov Xd, Vn.D[1]`: the upper doubleword of the vector. + if word & 0xfffe_fc00 == 0x9eae_0000 { + return Ok(SimdFmov(self::SimdFmov { + rd, + rn, + to_fp: word & 0x0001_0000 != 0, + double: true, + upper: true, + })); } // Across-lanes reductions: bits 21:17 = 11000, bits 11:10 = 10. They share // the three-same class bits, so they are matched first. ADDV (U=0, opcode @@ -1999,9 +2474,6 @@ pub fn decode(word: u32) -> Result { let q = word & 0x4000_0000 != 0; let u = word & 0x2000_0000 != 0; let size = ((word >> 22) & 3) as u8; - if !q && size == 3 { - return Err(UnsupportedInstruction); - } let opc = ((word >> 11) & 31) as u8; let op = match (u, opc) { (false, 0b10000) => Add, @@ -2054,6 +2526,35 @@ pub fn decode(word: u32) -> Result { (true, 0b00011) if size == 3 => Bif, _ => return Err(UnsupportedInstruction), }; + // `orn` and `bif` use `size` as an opcode; every other 64-bit lane needs + // the full vector and exists only in these groups. + if size == 3 + && !matches!(op, Orn | Bif) + && !(q + && matches!( + op, + Add | Sub + | CmEq + | CmGt + | CmGe + | CmHi + | CmHs + | CmTst + | SSHl + | USHl + | SQShl + | UQShl + | SQRShl + | UQRShl + | SQAdd + | UQAdd + | SQSub + | UQSub + | AddP + )) + { + return Err(UnsupportedInstruction); + } return Ok(SimdAlu(self::SimdAlu { op, rd, @@ -2066,7 +2567,7 @@ pub fn decode(word: u32) -> Result { // Table vector lookup: `tbl` and `tbx` over one to four table registers. // Bits 14:13 hold the table length, bit 12 selects tbl/tbx, and bits 11:10 // are reserved, so the class mask covers everything else. - if word & 0xbf20_8c00 == 0x0e00_0000 { + if word & 0xbfe0_8c00 == 0x0e00_0000 { return Ok(SimdTable(self::SimdTable { rd: (word & 0x1f) as u8, rn: ((word >> 5) & 0x1f) as u8, @@ -2076,29 +2577,33 @@ pub fn decode(word: u32) -> Result { q: word & 0x4000_0000 != 0, })); } - // `mov .[index], ` and the `umov`/`mov` lane-to-general form. - if word & 0xbf80_fc00 == 0x0e00_1c00 || word & 0xbf80_fc00 == 0x0e00_3c00 { + // `mov .[index], ` (`ins`, general): only the 128-bit form exists. + if word & 0xffe0_fc00 == 0x4e00_1c00 { let imm5 = (word >> 16) & 0x1f; if imm5 != 0 { let esize = imm5.trailing_zeros() as u8; - let index = (imm5 >> (esize + 1)) as u8; - let q = word & 0x4000_0000 != 0; - if esize <= 3 && (q || esize < 3) { - return if word & 0x0000_2000 == 0 { - Ok(SimdInsert(self::SimdInsert { - rd, - rn, - esize, - index, - })) - } else { - Ok(SimdExtract(self::SimdExtract { - rd, - rn, - esize, - index, - })) - }; + if esize <= 3 { + return Ok(SimdInsert(self::SimdInsert { + rd, + rn, + esize, + index: (imm5 >> (esize + 1)) as u8, + })); + } + } + } + // `umov`/`mov , .[index]`: `Q` is set exactly for the doubleword lane. + if word & 0xbfe0_fc00 == 0x0e00_3c00 { + let imm5 = (word >> 16) & 0x1f; + if imm5 != 0 { + let esize = imm5.trailing_zeros() as u8; + if esize <= 3 && (word & 0x4000_0000 != 0) == (esize == 3) { + return Ok(SimdExtract(self::SimdExtract { + rd, + rn, + esize, + index: (imm5 >> (esize + 1)) as u8, + })); } } } @@ -2119,7 +2624,7 @@ pub fn decode(word: u32) -> Result { } // `dup` from an element: the imm5 field encodes the element size as the // position of its lowest set bit and the index in the bits above. - if word & 0xbf80_fc00 == 0x0e00_0400 { + if word & 0xbfe0_fc00 == 0x0e00_0400 { let q = word & 0x4000_0000 != 0; let imm5 = (word >> 16) & 0x1f; if imm5 != 0 { @@ -2135,7 +2640,8 @@ pub fn decode(word: u32) -> Result { } } } - if word & 0xbf80_fc00 == 0x0e00_0c00 { + // `dup ., `: the doubleword arrangement needs the full vector. + if word & 0xbfe0_fc00 == 0x0e00_0c00 { let q = (word >> 30) & 1 != 0; let imm5 = (word >> 16) & 0x1f; if imm5 != 0 { @@ -2147,13 +2653,6 @@ pub fn decode(word: u32) -> Result { } } } - if word & 0xbef0_fc00 == 0x0ea0_1c00 { - let rn = ((word >> 5) & 0x1f) as u8; - let rm = ((word >> 16) & 0x1f) as u8; - if rn == rm { - return Ok(SimdMove { rd, rn }); - } - } if word & 0xfc00_0000 == 0x1400_0000 { return Ok(B(sign_extend(word, 26) << 2)); } @@ -2168,11 +2667,16 @@ pub fn decode(word: u32) -> Result { })); } if word & 0xbfe0_8400 == 0x2e00_0000 { - let rm = ((word >> 16) & 0x1f) as u8; + let q = word & 0x4000_0000 != 0; let imm = ((word >> 11) & 0xf) as u8; + // The 64-bit form extracts from a 16-byte pair, so the index is below 8. + if !q && imm >= 8 { + return Err(UnsupportedInstruction); + } + let rm = ((word >> 16) & 0x1f) as u8; let rn = ((word >> 5) & 0x1f) as u8; let rd = (word & 0x1f) as u8; - return Ok(SimdExt(self::SimdExt { rd, rn, rm, imm })); + return Ok(SimdExt(self::SimdExt { rd, rn, rm, imm, q })); } if word & 0x7fff_0000 == 0x5ac0_0000 { let op = match (word >> 10) & 63 { @@ -2247,8 +2751,11 @@ pub fn decode(word: u32) -> Result { rd, })); } + // SVC #imm16: the immediate is the whole field. + if word & 0xffe0_001f == 0xd400_0001 { + return Ok(Svc((word >> 5) as u16)); + } match word { - 0xd400_0001 => Ok(Svc), 0xd400_0002 => Ok(Psci), 0xd503_207f => Ok(Wfi), _ => Err(UnsupportedInstruction), @@ -2271,7 +2778,7 @@ mod tests { Addressing::Register { rm: 6, extend: Extend::Uxtx, - amount: 0, + amount: None, }, ), ( @@ -2282,13 +2789,13 @@ mod tests { Addressing::Register { rm: 6, extend: Extend::Uxtx, - amount: 3, + amount: Some(3), }, ), (0xf8410d07, Size::Double, 8, 7, Addressing::PreIndex(16)), (0xf8410549, Size::Double, 10, 9, Addressing::PostIndex(16)), - (0xf85f818b, Size::Double, 12, 11, Addressing::Offset(-8)), - (0xb85fc1cd, Size::Word, 14, 13, Addressing::Offset(-4)), + (0xf85f818b, Size::Double, 12, 11, Addressing::Unscaled(-8)), + (0xb85fc1cd, Size::Word, 14, 13, Addressing::Unscaled(-4)), ]; for (word, size, rn, rt, addressing) in rows { assert_eq!( @@ -2299,7 +2806,8 @@ mod tests { rn, rt, addressing, - signed: SignExtend::None + signed: SignExtend::None, + ordered: None, })) ); } @@ -2310,8 +2818,9 @@ mod tests { size: Size::Word, rn: 7, rt: 6, - addressing: Addressing::Offset(-4), - signed: SignExtend::None + addressing: Addressing::Unscaled(-4), + signed: SignExtend::None, + ordered: None, })) ); for (word, op, size, rn, rt, rt2, addressing) in [ @@ -2397,7 +2906,9 @@ mod tests { rt, rt2, addressing, - signed: SignExtend::None + signed: SignExtend::None, + // 0xa8300c02 is `stnp x2, x3, [x0, #-256]`; the rest are `ldp`/`stp`. + non_temporal: word == 0xa8300c02, })) ); } @@ -2435,7 +2946,7 @@ mod tests { read: true, register, rt: 0, - immediate: ((word >> 8) & 15) as u8 + immediate: 0 })) ); } @@ -2453,7 +2964,7 @@ mod tests { read: false, register, rt: 0, - immediate: ((word >> 8) & 15) as u8 + immediate: 0 })) ); } @@ -2474,38 +2985,238 @@ mod tests { ); } #[test] - fn assembler_hints_barriers_and_traps() { - use Instruction::*; - for (word, expected) in [ - (0xd5033f9f, Dsb), - (0xd5033fbf, Dmb), - (0xd5033fdf, Isb), - (0xd503201f, Nop), - (0xd503203f, Yield), - (0xd503205f, Wfe), - (0xd503207f, Wfi), - (0xd503209f, Sev), - (0xd50320bf, Sevl), - (0xd503241f, Nop), - (0xd503245f, Nop), - (0xd503249f, Nop), - (0xd50324df, Nop), - (0xd69f03e0, Eret), - (0xd4000001, Svc), - (0xd4000002, Psci), - (0xd4200020, Trap), - ] { - assert_eq!(decode(word), Ok(expected)); - } - assert_eq!(decode(0xd50b7423), Ok(DcZva(3))); - assert_eq!(decode(0xd503309f), Ok(Dsb)); - assert!(!Dsb.terminates()); - assert!(Isb.terminates()); - assert!(Eret.terminates()); + fn arithmetic_immediate_keeps_its_shift() { + let imm = |op, rn, rd, width, immediate, shifted| { + Ok(Instruction::ArithImm(Arithmetic { + op, + flags: false, + width, + rn, + rd, + immediate, + shifted, + })) + }; + // `add x0, x1, #0`, `add x0, x1, #0, lsl #12`, `add sp, x1, #0, lsl #12`. + assert_eq!( + decode(0x91000020), + imm(ArithOp::Add, 1, 0, Width::X64, 0, false) + ); + assert_eq!( + decode(0x91400020), + imm(ArithOp::Add, 1, 0, Width::X64, 0, true) + ); + assert_eq!( + decode(0x9140003f), + imm(ArithOp::Add, 1, 31, Width::X64, 0, true) + ); + // `add w0, w1, #3, lsl #12`, `sub x0, x1, #5, lsl #12`. + assert_eq!( + decode(0x11400c20), + imm(ArithOp::Add, 1, 0, Width::W32, 3, true) + ); + assert_eq!( + decode(0xd1401420), + imm(ArithOp::Sub, 1, 0, Width::X64, 5, true) + ); + let Ok(Instruction::ArithImm(a)) = decode(0x917ffc20) else { + panic!("add x0, x1, #0xfff, lsl #12 decodes"); + }; + assert_eq!((a.immediate, a.operand()), (0xfff, 0xfff000)); } + #[test] - fn logical_immediates_wrap_and_replicate_without_crossing_elements() { - for (word, immediate) in [ + fn debug_and_apple_zero_registers_keep_their_coordinates() { + use SystemRegister::*; + for (word, register) in [ + (0xd5300080, RazWi { crm: 0, op2: 4 }), + (0xd53000e0, RazWi { crm: 0, op2: 7 }), + (0xd5300fe0, RazWi { crm: 15, op2: 7 }), + ( + 0xd53df600, + AppleZero { + op1: 5, + crm: 6, + op2: 0, + }, + ), + ( + 0xd53ff680, + AppleZero { + op1: 7, + crm: 6, + op2: 4, + }, + ), + ( + 0xd539fd00, + AppleZero { + op1: 1, + crm: 13, + op2: 0, + }, + ), + ( + 0xd53df120, + AppleZero { + op1: 5, + crm: 1, + op2: 1, + }, + ), + ] { + assert_eq!( + decode(word), + Ok(Instruction::System(System { + read: true, + register, + rt: 0, + immediate: 0 + })), + "{word:#010x}" + ); + } + } + + #[test] + fn authenticated_branch_space_is_not_plain_branches() { + // Opcode bit 24 set with no authentication operand: objdump prints `.inst ...; undefined`. + for word in [0xd71f0000, 0xd73f0000, 0xd75f0000, 0xd79f03e0, 0xd71f03e0] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:#010x}" + ); + } + assert!(matches!(decode(0xd61f0000), Ok(Instruction::Indirect(_)))); + assert!(matches!(decode(0xd69f03e0), Ok(Instruction::Eret))); + } + + #[test] + fn msr_immediate_needs_rt_31_and_a_valid_immediate() { + // `msr pan, #0` and `msr daifset, #0` with Rt = 0, PAN/SPSel with CRm >= 2 (objdump + // spells these `msr s0_..., x0`). + for word in [ + 0xd5004080, 0xd50041a0, 0xd50340c0, 0xd50342e0, 0xd500429f, 0xd50042bf, 0xd50043bf, + 0xd500439f, + ] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:#010x}" + ); + } + for (word, register, immediate) in [ + (0xd500409f, SystemRegister::PanSet, 0), + (0xd500419f, SystemRegister::PanSet, 1), + (0xd50040bf, SystemRegister::SpSelSet, 0), + (0xd50041bf, SystemRegister::SpSelSet, 1), + (0xd50340df, SystemRegister::DaifSet, 0), + (0xd5034fdf, SystemRegister::DaifSet, 15), + (0xd50342ff, SystemRegister::DaifClear, 2), + ] { + assert_eq!( + decode(word), + Ok(Instruction::System(System { + read: false, + register, + rt: 31, + immediate + })), + "{word:#010x}" + ); + } + } + + #[test] + fn assembler_hints_barriers_and_traps() { + use Instruction::*; + for (word, expected) in [ + (0xd5033f9f, Dsb(0xf)), + (0xd5033fbf, Dmb(0xf)), + (0xd5033fdf, Isb(0xf)), + (0xd503201f, Nop), + (0xd503305f, Clrex(0)), + (0xd503203f, Yield), + (0xd503205f, Wfe), + (0xd503207f, Wfi), + (0xd503209f, Sev), + (0xd50320bf, Sevl), + (0xd503241f, Hint(32)), + (0xd503245f, Hint(34)), + (0xd503249f, Hint(36)), + (0xd50324df, Hint(38)), + (0xd69f03e0, Eret), + (0xd4000001, Svc(0)), + (0xd4001001, Svc(0x80)), + (0xd4000002, Psci), + (0xd4200020, Brk(1)), + (0xd4210000, Brk(0x800)), + ] { + assert_eq!(decode(word), Ok(expected)); + } + assert_eq!(decode(0xd50b7423), Ok(DcZva(3))); + assert_eq!(decode(0xd503309f), Ok(Dsb(0))); + } + #[test] + fn prefetch_and_system_operations_keep_their_operands() { + use Instruction::*; + assert_eq!( + decode(0xf9800420), + Ok(Prefetch(super::Prefetch { + op: 0, + rn: 1, + addressing: Addressing::Offset(8), + })) + ); + assert_eq!( + decode(0xf89f8040), + Ok(Prefetch(super::Prefetch { + op: 0, + rn: 2, + addressing: Addressing::Unscaled(-8), + })) + ); + assert_eq!( + decode(0xf8a47860), + Ok(Prefetch(super::Prefetch { + op: 0, + rn: 3, + addressing: Addressing::Register { + rm: 4, + extend: Extend::Uxtx, + amount: Some(3), + }, + })) + ); + assert_eq!( + decode(0xd8000040), + Ok(PrefetchLiteral(super::PrefetchLiteral { op: 0, offset: 8 })) + ); + assert_eq!( + decode(0xd50b7e20), + Ok(CacheOp(super::SysOp { + op1: 3, + crn: 7, + crm: 14, + op2: 1, + rt: 0, + })) + ); + assert_eq!( + decode(0xd508871f), + Ok(Tlbi(super::SysOp { + op1: 0, + crn: 8, + crm: 7, + op2: 0, + rt: 31, + })) + ); + } + #[test] + fn logical_immediates_wrap_and_replicate_without_crossing_elements() { + for (word, immediate) in [ (0x121c3c21, 0xffff0), (0x121c6c21, 0xfffffff0), (0x121c7021, 0xfffffff1), @@ -2584,9 +3295,9 @@ mod tests { 0x6f000422, 0xffff_fffe_ffff_fffe, 0xffff_fffe_ffff_fffe, - Set, + SetInverted, ), - (0x2f006489, 0xfbff_ffff_fbff_ffff, 0, Set), + (0x2f006489, 0xfbff_ffff_fbff_ffff, 0, SetInverted), ( 0x4f00a468, 0x0300_0300_0300_0300, @@ -2642,7 +3353,59 @@ mod tests { // modified immediate but have a non-zero `immh`; they are not `movi`. for word in [0x2f08a401, 0x4f255420, 0x4f090420, 0x0f0c8420] { assert!( - matches!(decode(word), Ok(Instruction::Host(_))), + !matches!(decode(word), Ok(Instruction::SimdImm(_))), + "{word:08x}" + ); + } + } + #[test] + fn simd_modified_immediates_keep_their_encoding_class() { + use ImmForm::*; + // (word, form, imm8, q); every word checked against GNU `objdump`. + for (word, form, imm8, q) in [ + (0x4f000400, Word { shift: 0 }, 0, true), + (0x0f000400, Word { shift: 0 }, 0, false), + (0x6f00e401, Double, 0, true), + (0x2f00e400, Double, 0, false), + (0x4f00e400, Byte, 0, true), + (0x0f00e460, Byte, 3, false), + (0x6f0707ff, Word { shift: 0 }, 0xff, true), + (0x6f00c400, WordMask { shift: 8 }, 0, true), + (0x4f00d420, WordMask { shift: 16 }, 1, true), + (0x0f006400, Word { shift: 24 }, 0, false), + (0x4f00a420, Half { shift: 8 }, 1, true), + (0x0f008420, Half { shift: 0 }, 1, false), + (0x4f00f400, FmovSingle, 0, true), + (0x6f00f400, FmovDouble, 0, true), + (0x4f00fc00, FmovHalf, 0, true), + (0x0f00fc00, FmovHalf, 0, false), + ] { + let Ok(Instruction::SimdImm(simd)) = decode(word) else { + panic!("{word:08x} is not a simd immediate") + }; + assert_eq!( + (simd.form, simd.imm8, simd.q), + (form, imm8, q), + "{word:08x}" + ); + } + // `mvni` and `movi` with the same pattern differ in `combine`. + let Ok(Instruction::SimdImm(mvni)) = decode(0x6f00045f) else { + panic!() + }; + assert_eq!(mvni.combine, ImmCombine::SetInverted); + // `fmov v.4h, #2.0` expands to the half-precision pattern in every lane. + let Ok(Instruction::SimdImm(half)) = decode(0x0f00fc00) else { + panic!() + }; + assert_eq!((half.low, half.high), (0x4000_4000_4000_4000, 0)); + // Unallocated in the modified-immediate class (`objdump`: `.inst`, undefined): + // bit 11 set outside the half-precision `fmov`, `fmov .2d` without `Q`, and + // `op` set with the half-precision form. + for word in [0x0f000ced, 0x4f00af48, 0x0f009c28, 0x2f00f420, 0x6f00fc20] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), "{word:08x}" ); } @@ -2708,6 +3471,57 @@ mod tests { assert_eq!(ext.rn, 29); assert_eq!(ext.rm, 0); assert_eq!(ext.imm, 8); + assert!(ext.q); + // `ext v0.8b, v1.8b, v2.8b, #7` has Q clear; with an index of 8 or more + // (`objdump`: undefined) there is no `.8b` form. + let Ok(Instruction::SimdExt(ext)) = decode(0x2e023820) else { + panic!("not ext") + }; + assert_eq!((ext.imm, ext.q), (7, false)); + assert_eq!(decode(0x2e164372), Err(DecodeError::UnsupportedInstruction)); + } + /// `objdump` calls every one of these `.inst ... ; undefined` or decodes them as a + /// different instruction (half-precision `fmla`/`fmulx`/`frecps`/`fmaxnm`, `luti2`, + /// `luti4`, indexed multiply); none may decode as the vector move, `tbl`, ALU or + /// immediate instruction whose mask they once satisfied. + #[test] + fn unallocated_or_other_simd_encodings_are_rejected() { + for word in [ + 0x4e5a0d5f, // fmla v31.8h (was `dup`) + 0x0e471c94, // fmulx v20.4h (was `mov v.b[n], w`) + 0x0e5b3cd8, // frecps v24.4h (was `umov`) + 0x4e4105db, // fmaxnm v27.8h (was `dup` element) + 0x4e8b21d7, // undefined table encoding + 0x4e86516b, // luti2 + 0x4e5c136f, // luti4 + 0x6eee6d74, // umin .2d + 0x4efb2521, // shsub .2d + 0x4ee6a4d4, // smaxp .2d + 0x4ef99d5a, // mul .2d + 0x0e1c1fba, // ins (general) with Q clear + 0x4e013c20, // umov w, v.b with Q set + 0x0e083c20, // umov x, v.d with Q clear + 0x4fab1d6c, // indexed-element class, bit 24 set + 0x2e5a0d5f, // fp16 class under U=1 + 0x6e000420, // dup/ins element with imm5 = 0 + ] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:08x}" + ); + } + } + #[test] + fn decodes_bitwise_three_same_with_the_64_bit_form_selected_by_size() { + // `bif v0.8b, v31.8b, v30.8b` and `orn v0.8b, v1.8b, v0.8b`: `size` is part of + // the opcode, so `size == 3` with `Q` clear is valid here. + for (word, op) in [(0x2efe1fe0, SimdAluOp::Bif), (0x0ee01c20, SimdAluOp::Orn)] { + let Ok(Instruction::SimdAlu(alu)) = decode(word) else { + panic!("{word:08x}") + }; + assert_eq!((alu.op, alu.size, alu.q), (op, 3, false), "{word:08x}"); + } } #[test] fn decodes_single_structure_simd_memory() { @@ -2719,6 +3533,7 @@ mod tests { rn: 1, rt: 0, addressing: Addressing::Offset(0), + access: SimdAccess::Ld1 { esize: 1 }, })) ); assert_eq!( @@ -2729,6 +3544,7 @@ mod tests { rn: 1, rt: 0, addressing: Addressing::PostIndex(16), + access: SimdAccess::Ld1 { esize: 1 }, })) ); } @@ -2755,6 +3571,7 @@ mod tests { rn, rt, rt2, + order: MemoryOrder::default(), })) }; // ldxp x9, x21, [x8]; stxp w11, x19, x10, [x8]; ldxp w0, wzr, [x0] @@ -3049,7 +3866,8 @@ mod tests { rd: 2, rn: 2, to_fp: false, - double: true + double: true, + upper: false, })) ); assert_eq!( @@ -3058,7 +3876,8 @@ mod tests { rd: 2, rn: 2, to_fp: true, - double: true + double: true, + upper: false, })) ); assert_eq!( @@ -3067,7 +3886,8 @@ mod tests { rd: 2, rn: 1, to_fp: false, - double: false + double: false, + upper: false, })) ); assert_eq!( @@ -3076,7 +3896,29 @@ mod tests { rd: 1, rn: 2, to_fp: true, - double: false + double: false, + upper: false, + })) + ); + // `fmov v0.d[1], x3` / `fmov x3, v0.d[1]` move the upper doubleword. + assert_eq!( + decode(0x9eaf0060), + Ok(Instruction::SimdFmov(SimdFmov { + rd: 0, + rn: 3, + to_fp: true, + double: true, + upper: true, + })) + ); + assert_eq!( + decode(0x9eae0003), + Ok(Instruction::SimdFmov(SimdFmov { + rd: 3, + rn: 0, + to_fp: false, + double: true, + upper: true, })) ); } @@ -3187,4 +4029,426 @@ mod tests { } } } + fn memory(op: MemoryOp, size: Size, rt: u8, rn: u8, addressing: Addressing) -> Memory { + Memory { + op, + size, + rn, + rt, + addressing, + signed: SignExtend::None, + ordered: None, + } + } + #[test] + fn ordered_memory_accesses_keep_their_class() { + use MemoryOp::{Load, Store}; + let ordered = |op, size, rt, rn, ordered| { + Ok(Instruction::Memory(Memory { + ordered: Some(ordered), + ..memory(op, size, rt, rn, Addressing::Offset(0)) + })) + }; + // ldar w0, [x1]; ldar x0, [sp]; ldarb w0, [x1]; stlr x0, [sp]; stlrh w0, [x1] + assert_eq!( + decode(0x88dffc20), + ordered(Load, Size::Word, 0, 1, Ordered::Acquire) + ); + assert_eq!( + decode(0xc8dfffe0), + ordered(Load, Size::Double, 0, 31, Ordered::Acquire) + ); + assert_eq!( + decode(0x08dffc20), + ordered(Load, Size::Byte, 0, 1, Ordered::Acquire) + ); + assert_eq!( + decode(0xc89fffe0), + ordered(Store, Size::Double, 0, 31, Ordered::Release) + ); + assert_eq!( + decode(0x489ffc20), + ordered(Store, Size::Half, 0, 1, Ordered::Release) + ); + // ldlar x0, [x1]; stllrb w0, [x1] + assert_eq!( + decode(0xc8df7c20), + ordered(Load, Size::Double, 0, 1, Ordered::LimitedAcquire) + ); + assert_eq!( + decode(0x089f7c20), + ordered(Store, Size::Byte, 0, 1, Ordered::LimitedRelease) + ); + // ldapr w0, [x1]; ldaprh w0, [x1] + assert_eq!( + decode(0xb8bfc020), + ordered(Load, Size::Word, 0, 1, Ordered::AcquirePc) + ); + assert_eq!( + decode(0x78bfc020), + ordered(Load, Size::Half, 0, 1, Ordered::AcquirePc) + ); + // The unscaled RCpc forms: ldapur w0, [x1, #4]; stlur w0, [x1, #4]; + // ldapursb x0, [x1, #4]. + assert_eq!( + decode(0x99404020), + Ok(Instruction::Memory(Memory { + ordered: Some(Ordered::AcquirePc), + ..memory(Load, Size::Word, 0, 1, Addressing::Unscaled(4)) + })) + ); + assert_eq!( + decode(0x99004020), + Ok(Instruction::Memory(Memory { + ordered: Some(Ordered::Release), + ..memory(Store, Size::Word, 0, 1, Addressing::Unscaled(4)) + })) + ); + assert_eq!( + decode(0x19804020), + Ok(Instruction::Memory(Memory { + signed: SignExtend::To64, + ordered: Some(Ordered::AcquirePc), + ..memory(Load, Size::Byte, 0, 1, Addressing::Unscaled(4)) + })) + ); + } + #[test] + fn exclusives_and_atomics_keep_acquire_and_release() { + let exclusive = |op, size, rs, rt, acquire, release| { + Ok(Instruction::Exclusive(Exclusive { + op, + size, + rn: 1, + rt, + rs, + order: MemoryOrder { acquire, release }, + })) + }; + // ldxr w0, [x1]; ldaxr w0, [x1]; stxr w2, w0, [x1]; stlxr w2, w0, [x1] + assert_eq!( + decode(0x885f7c20), + exclusive(MemoryOp::Load, Size::Word, 31, 0, false, false) + ); + assert_eq!( + decode(0x885ffc20), + exclusive(MemoryOp::Load, Size::Word, 31, 0, true, false) + ); + assert_eq!( + decode(0x88027c20), + exclusive(MemoryOp::Store, Size::Word, 2, 0, false, false) + ); + assert_eq!( + decode(0x8802fc20), + exclusive(MemoryOp::Store, Size::Word, 2, 0, false, true) + ); + let atomic = |kind, size, rs, rt, rt2, acquire, release| { + Ok(Instruction::Atomic(Atomic { + kind, + size, + rs, + rn: 2, + rt, + rt2, + order: MemoryOrder { acquire, release }, + })) + }; + // cas w0, w1, [x2]; casa; casl; casal x0, x1, [x2] + for (word, size, acquire, release) in [ + (0x88a07c41, Size::Word, false, false), + (0x88e07c41, Size::Word, true, false), + (0x88a0fc41, Size::Word, false, true), + (0xc8e0fc41, Size::Double, true, true), + ] { + assert_eq!( + decode(word), + atomic(AtomicKind::Cas, size, 0, 1, 0, acquire, release), + "{word:#010x}" + ); + } + // casp, caspa, caspl, caspal x0, x1, x2, x3, [x4] share one kind. + let casp = |word, acquire, release| { + assert_eq!( + decode(word), + Ok(Instruction::Atomic(Atomic { + kind: AtomicKind::Casp, + size: Size::Double, + rs: 0, + rn: 4, + rt: 2, + rt2: 0, + order: MemoryOrder { acquire, release }, + })), + "{word:#010x}" + ); + }; + casp(0x48207c82, false, false); + casp(0x48607c82, true, false); + casp(0x4820fc82, false, true); + casp(0x4860fc82, true, true); + // swpal x0, xzr, [x2]; ldadda w0, wzr, [x1] and ldaddl w0, w1, [x2] + assert_eq!( + decode(0xf8e0805f), + atomic(AtomicKind::Swap, Size::Double, 0, 31, 0, true, true) + ); + assert_eq!( + decode(0xb8a0003f), + Ok(Instruction::Atomic(Atomic { + kind: AtomicKind::Add, + size: Size::Word, + rs: 0, + rn: 1, + rt: 31, + rt2: 0, + order: MemoryOrder { + acquire: true, + release: false + }, + })) + ); + assert_eq!( + decode(0xb8600041), + atomic(AtomicKind::Add, Size::Word, 0, 1, 0, false, true) + ); + // ldaxp w0, w1, [x2]; stlxp w3, x0, x1, [x2] + assert_eq!( + decode(0x887f8440), + atomic(AtomicKind::LoadPair, Size::Word, 31, 0, 1, true, false) + ); + assert_eq!( + decode(0xc8238440), + atomic(AtomicKind::StorePair, Size::Double, 3, 0, 1, false, true) + ); + } + #[test] + fn offset_forms_keep_their_encoding() { + use MemoryOp::Load; + // ldr x0, [x1, #8]; ldur x0, [x1, #8]; ldtr w0, [x1, #4]; ldur w0, [x1] + assert_eq!( + decode(0xf9400420), + Ok(Instruction::Memory(memory( + Load, + Size::Double, + 0, + 1, + Addressing::Offset(8) + ))) + ); + assert_eq!( + decode(0xf8408020), + Ok(Instruction::Memory(memory( + Load, + Size::Double, + 0, + 1, + Addressing::Unscaled(8) + ))) + ); + assert_eq!( + decode(0xb8404820), + Ok(Instruction::Memory(memory( + Load, + Size::Word, + 0, + 1, + Addressing::Unprivileged(4) + ))) + ); + assert_eq!( + decode(0xb8400020), + Ok(Instruction::Memory(memory( + Load, + Size::Word, + 0, + 1, + Addressing::Unscaled(0) + ))) + ); + // ldr q0, [x1]; ldur q0, [x1, #16]; ldur b0, [x1, #1]; ldr b0, [x1, #1] + let simd = |word, bytes, addressing| { + assert_eq!( + decode(word), + Ok(Instruction::SimdMemory(SimdMemory { + op: MemoryOp::Load, + bytes, + rn: 1, + rt: 0, + addressing, + access: SimdAccess::Register, + })), + "{word:#010x}" + ); + }; + simd(0x3dc00020, 16, Addressing::Offset(0)); + simd(0x3cc10020, 16, Addressing::Unscaled(16)); + simd(0x3c401020, 1, Addressing::Unscaled(1)); + simd(0x3d400420, 1, Addressing::Offset(1)); + // prfm pldl1keep, [x1, #8]; prfum pldl1keep, [x1, #8] + assert_eq!( + decode(0xf9800420), + Ok(Instruction::Prefetch(Prefetch { + op: 0, + rn: 1, + addressing: Addressing::Offset(8) + })) + ); + assert_eq!( + decode(0xf8808020), + Ok(Instruction::Prefetch(Prefetch { + op: 0, + rn: 1, + addressing: Addressing::Unscaled(8), + })) + ); + } + #[test] + fn register_offsets_keep_whether_they_are_shifted() { + let register = |word, extend, amount| { + assert_eq!( + decode(word), + Ok(Instruction::Memory(memory( + MemoryOp::Load, + Size::Byte, + 0, + 1, + Addressing::Register { + rm: 2, + extend, + amount + } + ))), + "{word:#010x}" + ); + }; + // ldrb w0, [x1, x2, lsl #0]; ldrb w0, [x1, x2]; ldrb w0, [x1, w2, sxtw #0]; + // ldrb w0, [x1, w2, sxtw] + register(0x38627820, Extend::Uxtx, Some(0)); + register(0x38626820, Extend::Uxtx, None); + register(0x3862d820, Extend::Sxtw, Some(0)); + register(0x3862c820, Extend::Sxtw, None); + // str b2, [x11, x0, lsl #0]; str q2, [x11, x0, lsl #4]; str q2, [x11, x0] + let simd = |word, bytes, amount| { + assert_eq!( + decode(word), + Ok(Instruction::SimdMemory(SimdMemory { + op: MemoryOp::Store, + bytes, + rn: 11, + rt: 2, + addressing: Addressing::Register { + rm: 0, + extend: Extend::Uxtx, + amount + }, + access: SimdAccess::Register, + })), + "{word:#010x}" + ); + }; + simd(0x3c207962, 1, Some(0)); + simd(0x3ca07962, 16, Some(4)); + simd(0x3ca06962, 16, None); + } + #[test] + fn non_temporal_pairs_keep_their_identity() { + let Ok(Instruction::Pair(pair)) = decode(0xa8400440) else { + panic!("ldnp x0, x1, [x2]"); + }; + assert_eq!( + (pair.op, pair.size, pair.addressing, pair.non_temporal), + (MemoryOp::Load, Size::Double, Addressing::Offset(0), true) + ); + let Ok(Instruction::Pair(pair)) = decode(0x28010440) else { + panic!("stnp w0, w1, [x2, #8]"); + }; + assert_eq!( + (pair.op, pair.size, pair.addressing, pair.non_temporal), + (MemoryOp::Store, Size::Word, Addressing::Offset(8), true) + ); + let Ok(Instruction::Pair(pair)) = decode(0xa9400440) else { + panic!("ldp x0, x1, [x2]"); + }; + assert!(!pair.non_temporal); + let Ok(Instruction::SimdPair(pair)) = decode(0xac3f0440) else { + panic!("stnp q0, q1, [x2, #-32]"); + }; + assert_eq!( + (pair.op, pair.bytes, pair.addressing, pair.non_temporal), + (MemoryOp::Store, 16, Addressing::Offset(-32), true) + ); + let Ok(Instruction::SimdPair(pair)) = decode(0xadc08440) else { + panic!("ldp q0, q1, [x2, #16]!"); + }; + assert!(!pair.non_temporal); + } + #[test] + fn one_register_ld1_keeps_its_arrangement() { + let ld1 = |word, op, bytes, addressing, esize| { + assert_eq!( + decode(word), + Ok(Instruction::SimdMemory(SimdMemory { + op, + bytes, + rn: 1, + rt: 0, + addressing, + access: SimdAccess::Ld1 { esize }, + })), + "{word:#010x}" + ); + }; + // ld1 {v0.8b}, [x1]; ld1 {v0.8h}, [x1]; ld1 {v0.2s}, [x1]; ld1 {v0.1d}, [x1] + ld1(0x0c407020, MemoryOp::Load, 8, Addressing::Offset(0), 1); + ld1(0x4c407420, MemoryOp::Load, 16, Addressing::Offset(0), 2); + ld1(0x0c407820, MemoryOp::Load, 8, Addressing::Offset(0), 4); + ld1(0x0c407c20, MemoryOp::Load, 8, Addressing::Offset(0), 8); + // ld1 {v0.4s}, [x1], #16; st1 {v0.2s}, [x1], #8 + ld1(0x4cdf7820, MemoryOp::Load, 16, Addressing::PostIndex(16), 4); + ld1(0x0c9f7820, MemoryOp::Store, 8, Addressing::PostIndex(8), 4); + // A register post-index is a structure access. + assert!(matches!(decode(0x4cc27020), Ok(Instruction::SimdStruct(_)))); + } + #[test] + fn range_prefetch_and_ldrsw_literal_decode() { + // rprfm pldkeep, x2, [x1]; rprfm pldstrm, xzr, [sp]; rprfm #2, x2, [x1] + for (word, op, rm, rn) in [ + (0xf8a24838, 0, 2, 1), + (0xf8bf4bfc, 4, 31, 31), + (0xf8a2483a, 2, 2, 1), + ] { + assert_eq!( + decode(word), + Ok(Instruction::RangePrefetch(RangePrefetch { op, rm, rn })), + "{word:#010x}" + ); + } + // ldrsw x0, #8 + assert_eq!( + decode(0x98000040), + Ok(Instruction::Literal(Literal { + size: Size::Word, + rt: 0, + offset: 8, + signed: SignExtend::To64, + })) + ); + } + #[test] + fn unallocated_memory_words_are_rejected() { + // Each is `.inst ... ; undefined` for GNU objdump 2.46: PRFM (register) with a + // byte or halfword extend, LDAPR with the wrong A/R bits or a non-zero Rs, an + // unprivileged SIMD&FP access, LDNP with the `ldpsw` opc, and LDAPURS* with an + // impossible size. + for word in [ + 0xf8a20838, 0xf8a21838, 0xf8a28838, 0xf8a29838, 0xf8a2a838, 0xf8a20837, 0xb8ffc020, + 0xb87fc020, 0xb83fc020, 0x78ffc020, 0xb8bdc020, 0x3c400820, 0x3c000820, 0x68400440, + 0x68000440, 0x99c04020, 0xd9804020, + ] { + assert_eq!( + decode(word), + Err(DecodeError::UnsupportedInstruction), + "{word:#010x}" + ); + } + } } diff --git a/src/isa/aarch64/src/flow.rs b/src/isa/aarch64/src/flow.rs new file mode 100644 index 0000000..3cfc298 --- /dev/null +++ b/src/isa/aarch64/src/flow.rs @@ -0,0 +1,137 @@ +//! Control-flow classification of A64 instructions, for analysis. +//! +//! [`Instruction::flow`] answers the CFG question: where can execution go +//! next. It is architectural. Whether a particular translator cuts its blocks +//! at loads, stores or system instructions is that translator's policy and is +//! not decided here. +use crate::decode::{IndirectKind, Instruction, decode, is_undefined}; + +/// Where execution can go after an instruction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Flow { + /// Continues at `pc + 4`. + Next, + /// Unconditional direct branch. + Jump(u64), + /// Conditional direct branch: taken target, otherwise `pc + 4`. + CondJump(u64), + /// Direct call (`bl`); returns to `pc + 4`. + Call(u64), + /// Indirect call (`blr`); returns to `pc + 4`. + CallIndirect, + /// Indirect jump (`br`); no known successor. + JumpIndirect, + /// `ret`, `eret`. + Return, + /// Does not continue (`brk`, permanently undefined words). + Stop, + /// The word is not an instruction this crate decodes, so where it goes is + /// unknown. Analysis that must keep walking treats it as falling through + /// (see [`Flow::falls_through`]); it never ends a block. + Unrecognized, +} + +impl Flow { + /// Whether the instruction after this one (`pc + 4`) is a successor. + pub fn falls_through(self) -> bool { + matches!( + self, + Self::Next + | Self::CondJump(_) + | Self::Call(_) + | Self::CallIndirect + | Self::Unrecognized + ) + } + /// The statically known branch or call target. + pub fn target(self) -> Option { + match self { + Self::Jump(t) | Self::CondJump(t) | Self::Call(t) => Some(t), + _ => None, + } + } + /// Whether this instruction ends a basic block. + pub fn ends_block(self) -> bool { + !matches!(self, Self::Next | Self::Unrecognized) + } +} + +impl Instruction { + /// Where execution can go after this instruction when it is located at `pc`. + pub fn flow(&self, pc: u64) -> Flow { + let rel = |off: i64| pc.wrapping_add(off as u64); + match self { + Self::B(o) => Flow::Jump(rel(*o)), + Self::BCond(b) => Flow::CondJump(rel(b.offset)), + Self::TestBranch(t) => Flow::CondJump(rel(t.offset)), + Self::Call(c) if c.link => Flow::Call(rel(c.target)), + Self::Call(c) => Flow::Jump(rel(c.target)), + Self::Indirect(i) => match i.kind { + IndirectKind::Jump => Flow::JumpIndirect, + IndirectKind::Call => Flow::CallIndirect, + IndirectKind::Return => Flow::Return, + }, + Self::Eret => Flow::Return, + Self::Brk(_) => Flow::Stop, + _ => Flow::Next, + } + } +} + +/// Classify the instruction `word` located at `pc`. Permanently undefined +/// words are [`Flow::Stop`]; words the decoder does not recognise are +/// [`Flow::Unrecognized`]. +pub fn flow(word: u32, pc: u64) -> Flow { + match decode(word) { + Ok(instruction) => instruction.flow(pc), + Err(_) if is_undefined(word) => Flow::Stop, + Err(_) => Flow::Unrecognized, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies() { + assert_eq!(flow(0xd65f_03c0, 0x100), Flow::Return); + assert_eq!(flow(0xd61f_0200, 0x100), Flow::JumpIndirect); // br x16 + assert_eq!(flow(0xd63f_0100, 0x100), Flow::CallIndirect); // blr x8 + assert_eq!(flow(0x1400_0004, 0x100), Flow::Jump(0x110)); // b .+16 + assert_eq!(flow(0x97ff_ffff, 0x100), Flow::Call(0xfc)); // bl .-4 + assert_eq!(flow(0x3400_00e8, 0x100), Flow::CondJump(0x11c)); // cbz w8, .+28 + assert_eq!(flow(0xd503_201f, 0x100), Flow::Next); // nop + assert_eq!(flow(0xd420_0020, 0x100), Flow::Stop); // brk #1 + assert_eq!(flow(0xd400_0021, 0x100), Flow::Next); // svc #1 + assert_eq!(flow(0xe7ff_deff, 0x100), Flow::Stop); // udf: permanently undefined + } + + #[test] + fn ret_and_br_stay_distinct_in_the_decoded_instruction() { + use crate::decode::Indirect; + let ret = decode(0xd65f_03c0).unwrap(); + let br = decode(0xd61f_0200).unwrap(); + assert_eq!( + ret, + Instruction::Indirect(Indirect { + rn: 30, + kind: IndirectKind::Return + }) + ); + assert_eq!( + br, + Instruction::Indirect(Indirect { + rn: 16, + kind: IndirectKind::Jump + }) + ); + } + + #[test] + fn unrecognized_words_neither_end_a_block_nor_stop_a_walk() { + assert!(Flow::Unrecognized.falls_through()); + assert!(!Flow::Unrecognized.ends_block()); + assert!(!Flow::Stop.falls_through()); + } +} diff --git a/src/isa/aarch64/src/format/fp.rs b/src/isa/aarch64/src/format/fp.rs new file mode 100644 index 0000000..8176276 --- /dev/null +++ b/src/isa/aarch64/src/format/fp.rs @@ -0,0 +1,357 @@ +//! Scalar floating point and the general-register / SIMD&FP register `fmov`. +//! +//! Owns `Fp` and `SimdFmov`. + +use super::Context; +use super::reg::{arrangement, fp_scalar, gpr}; +use crate::decode::{Instruction, SimdFmov, Width}; +use crate::fp::{Binary, Fused, Op, Precision, Unary}; +use core::fmt::{self, Formatter}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + let _ = cx; + match instruction { + Instruction::Fp(op) => write_op(f, *op), + Instruction::SimdFmov(op) => write_fmov(f, *op), + other => unreachable!("{other:?} is not a floating-point instruction"), + } +} + +/// Size in bytes of the scalar register for a precision. +fn size(p: Precision) -> u8 { + match p { + Precision::Single => 4, + Precision::Double => 8, + } +} + +fn width(wide: bool) -> Width { + if wide { Width::X64 } else { Width::W32 } +} + +/// The condition field as binutils spells it; the field is four bits. +fn condition(f: &mut Formatter<'_>, cond: u32) -> fmt::Result { + const NAMES: [&str; 16] = [ + "eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc", "hi", "ls", "ge", "lt", "gt", "le", "al", + "nv", + ]; + f.write_str(NAMES[(cond & 15) as usize]) +} + +fn write_fmov(f: &mut Formatter<'_>, op: SimdFmov) -> fmt::Result { + f.write_str("fmov ")?; + // The upper doubleword is named as the lane `Vn.D[1]`. + let vector = |f: &mut Formatter<'_>, r: u8| -> fmt::Result { + if op.upper { + write!(f, "v{r}.d[1]") + } else { + fp_scalar(f, if op.double { 8 } else { 4 }, r) + } + }; + if op.to_fp { + vector(f, op.rd)?; + f.write_str(", ")?; + gpr(f, width(op.double), op.rn) + } else { + gpr(f, width(op.double), op.rd)?; + f.write_str(", ")?; + vector(f, op.rn) + } +} + +/// `mnemonic rd, rn` over scalar registers of precisions `pd` and `pn`. +fn two( + f: &mut Formatter<'_>, + mnemonic: &str, + (pd, rd): (Precision, u8), + (pn, rn): (Precision, u8), +) -> fmt::Result { + write!(f, "{mnemonic} ")?; + fp_scalar(f, size(pd), rd)?; + f.write_str(", ")?; + fp_scalar(f, size(pn), rn) +} + +fn regs(f: &mut Formatter<'_>, p: Precision, list: &[u8]) -> fmt::Result { + for (i, &r) in list.iter().enumerate() { + if i != 0 { + f.write_str(", ")?; + } + fp_scalar(f, size(p), r)?; + } + Ok(()) +} + +fn write_op(f: &mut Formatter<'_>, op: Op) -> fmt::Result { + match op { + Op::Unary { kind, p, rd, rn } => { + let mnemonic = match kind { + Unary::Move => "fmov", + Unary::Abs => "fabs", + Unary::Neg => "fneg", + Unary::Sqrt => "fsqrt", + }; + two(f, mnemonic, (p, rd), (p, rn)) + } + Op::Convert { p, rd, rn } => { + let to = match p { + Precision::Single => Precision::Double, + Precision::Double => Precision::Single, + }; + two(f, "fcvt", (to, rd), (p, rn)) + } + Op::Binary { + kind, + p, + rd, + rn, + rm, + } => { + let mnemonic = match kind { + Binary::Mul => "fmul", + Binary::Div => "fdiv", + Binary::Add => "fadd", + Binary::Sub => "fsub", + Binary::Max => "fmax", + Binary::Min => "fmin", + Binary::MaxNum => "fmaxnm", + Binary::MinNum => "fminnm", + Binary::Nmul => "fnmul", + }; + write!(f, "{mnemonic} ")?; + regs(f, p, &[rd, rn, rm]) + } + Op::Fused { + kind, + p, + rd, + rn, + rm, + ra, + } => { + let mnemonic = match kind { + Fused::Madd => "fmadd", + Fused::Msub => "fmsub", + Fused::Nmadd => "fnmadd", + Fused::Nmsub => "fnmsub", + }; + write!(f, "{mnemonic} ")?; + regs(f, p, &[rd, rn, rm, ra]) + } + Op::Compare { + p, + rn, + rm, + signaling, + } => { + f.write_str(if signaling { "fcmpe " } else { "fcmp " })?; + fp_scalar(f, size(p), rn)?; + f.write_str(", ")?; + match rm { + Some(rm) => fp_scalar(f, size(p), rm), + None => f.write_str("#0.0"), + } + } + Op::CondCompare { + p, + rn, + rm, + cond, + nzcv, + } => { + f.write_str("fccmp ")?; + regs(f, p, &[rn, rm])?; + write!(f, ", #{:#x}, ", nzcv & 15)?; + condition(f, cond) + } + Op::Select { + p, + rd, + rn, + rm, + cond, + } => { + f.write_str("fcsel ")?; + regs(f, p, &[rd, rn, rm])?; + f.write_str(", ")?; + condition(f, cond) + } + Op::Immediate { p, rd, imm8 } => { + f.write_str("fmov ")?; + fp_scalar(f, size(p), rd)?; + f.write_str(", #")?; + expanded_immediate(f, imm8 as u8) + } + Op::IntToFloat { + signed, + p, + wide, + rd, + rn, + } => { + f.write_str(if signed { "scvtf " } else { "ucvtf " })?; + fp_scalar(f, size(p), rd)?; + f.write_str(", ")?; + gpr(f, width(wide), rn) + } + Op::FloatToInt { + signed, + p, + wide, + rd, + rn, + } => { + f.write_str(if signed { "fcvtzs " } else { "fcvtzu " })?; + gpr(f, width(wide), rd)?; + f.write_str(", ")?; + fp_scalar(f, size(p), rn) + } + Op::ScalarIntToFloat { signed, p, rd, rn } => { + two(f, if signed { "scvtf" } else { "ucvtf" }, (p, rd), (p, rn)) + } + Op::ScalarFloatToInt { signed, p, rd, rn } => two( + f, + if signed { "fcvtzs" } else { "fcvtzu" }, + (p, rd), + (p, rn), + ), + Op::VectorSign { + negate, + double, + q, + rd, + rn, + } => { + f.write_str(if negate { "fneg " } else { "fabs " })?; + let esize = if double { 3 } else { 2 }; + write!(f, "v{rd}.")?; + arrangement(f, esize, q)?; + write!(f, ", v{rn}.")?; + arrangement(f, esize, q) + } + } +} + +/// Print the VFPExpandImm value of `imm8` the way binutils does: `%.18e`. +/// +/// With `imm8 = a:bcd:efgh` the value is `±(16 + efgh) / 16 * 2^exp` where `exp` is +/// the signed three-bit `NOT(b):c:d` plus one (`-3..=4`). That is `n / 2^j` with +/// `n` in `16..=31` and `j` in `0..=7`, so `n * 5^j / 10^j` is an exact decimal +/// integer of at most seven digits. +fn expanded_immediate(f: &mut Formatter<'_>, imm8: u8) -> fmt::Result { + let negative = imm8 & 0x80 != 0; + let e = i32::from((imm8 >> 4) & 7); + let exp = if e < 4 { e + 1 } else { e - 7 }; + let n = 16 + u32::from(imm8 & 15); + let j = (4 - exp) as u32; + // `j` is `0..=7`; `n * 5^j` is at most 31 * 78125. + let digits_value = n * 5u32.pow(j); + let mut digits = [0u8; 10]; + let mut count = 0; + let mut rest = digits_value; + while rest != 0 { + digits[count] = (rest % 10) as u8; + rest /= 10; + count += 1; + } + // `digits[count - 1]` is the most significant digit. + let decimal_exponent = count as i32 - 1 - j as i32; + if negative { + f.write_str("-")?; + } + write!(f, "{}.", digits[count - 1])?; + for i in 1..19 { + let d = if i < count { digits[count - 1 - i] } else { 0 }; + write!(f, "{d}")?; + } + let sign = if decimal_exponent < 0 { '-' } else { '+' }; + write!(f, "e{sign}{:02}", decimal_exponent.unsigned_abs()) +} + +#[cfg(test)] +mod tests { + use crate::format::word; + use core::fmt::Write; + + struct Buf { + bytes: [u8; 96], + len: usize, + } + + impl Write for Buf { + fn write_str(&mut self, s: &str) -> core::fmt::Result { + self.bytes[self.len..self.len + s.len()].copy_from_slice(s.as_bytes()); + self.len += s.len(); + Ok(()) + } + } + + fn check(w: u32, expected: &str) { + let mut buf = Buf { + bytes: [0; 96], + len: 0, + }; + write!(buf, "{}", word(w)).unwrap(); + assert_eq!( + core::str::from_utf8(&buf.bytes[..buf.len]).unwrap(), + expected, + "{w:#010x}" + ); + } + + #[test] + fn scalar_floating_point() { + check(0x1e602008, "fcmp d0, #0.0"); + check(0x1e222020, "fcmp s1, s2"); + check(0x1e222030, "fcmpe s1, s2"); + check(0x1e202038, "fcmpe s1, #0.0"); + check(0x1e7f2010, "fcmpe d0, d31"); + check(0x1e221425, "fccmp s1, s2, #0x5, ne"); + check(0x1e60e7ef, "fccmp d31, d0, #0xf, al"); + check(0x1e633c41, "fcsel d1, d2, d3, cc"); + check(0x1e22c020, "fcvt d0, s1"); + check(0x1e6243e0, "fcvt s0, d31"); + check(0x1e638841, "fnmul d1, d2, d3"); + check(0x1f239041, "fnmsub s1, s2, s3, s4"); + check(0x1e204041, "fmov s1, s2"); + check(0x1e21c041, "fsqrt s1, s2"); + check(0x1e636841, "fmaxnm d1, d2, d3"); + } + + #[test] + fn fmov_to_and_from_the_upper_doubleword() { + check(0x9eaf0060, "fmov v0.d[1], x3"); + check(0x9eae0003, "fmov x3, v0.d[1]"); + check(0x9e670042, "fmov d2, x2"); + } + + #[test] + fn immediates() { + check(0x1e78101f, "fmov d31, #-1.250000000000000000e-01"); + check(0x1e27f000, "fmov s0, #3.100000000000000000e+01"); + check(0x1e281000, "fmov s0, #1.250000000000000000e-01"); + } + + #[test] + fn conversions() { + check(0x1e620041, "scvtf d1, w2"); + check(0x9e230041, "ucvtf s1, x2"); + check(0x1e780020, "fcvtzs w0, d1"); + check(0x9e390020, "fcvtzu x0, s1"); + check(0x5e21d841, "scvtf s1, s2"); + check(0x7e61d841, "ucvtf d1, d2"); + check(0x5ee1b841, "fcvtzs d1, d2"); + check(0x7ea1b841, "fcvtzu s1, s2"); + check(0x0ea0f841, "fabs v1.2s, v2.2s"); + check(0x6ea0f841, "fneg v1.4s, v2.4s"); + check(0x4ee0f841, "fabs v1.2d, v2.2d"); + check(0x9e670020, "fmov d0, x1"); + check(0x9e660020, "fmov x0, d1"); + check(0x1e270020, "fmov s0, w1"); + check(0x1e260020, "fmov w0, s1"); + } +} diff --git a/src/isa/aarch64/src/format/integer.rs b/src/isa/aarch64/src/format/integer.rs new file mode 100644 index 0000000..502a758 --- /dev/null +++ b/src/isa/aarch64/src/format/integer.rs @@ -0,0 +1,877 @@ +//! Integer data processing: moves, add/subtract, logical, multiply, divide, shifts, +//! bitfields, conditional select/compare, `adr`/`adrp`. +//! +//! Owns `Movz`, `Movn`, `Movk`, `ArithImm`, `ArithReg`, `ArithExt`, `LogicReg`, +//! `LogicImm`, `Mul`, `MulLong`, `MulHigh`, `Adr`, `Csel`, `Variable`, `CondCompare`, +//! `ArithCarry`, `Unary`, `Bitfield` and `Extract`. + +use super::Context; +use super::reg::{condition, gpr, gpr_sp}; +use crate::decode::{ + AddCarry, Address, ArithOp, Arithmetic, ArithmeticExtended, ArithmeticReg, Bitfield, + BitfieldOp, CompareOperand, CondCompare, Condition, Extend, Extract, Instruction, Logic, + LogicImm, LogicOp, Multiply, MultiplyHigh, MultiplyLong, Select, Shift, Unary, UnaryOp, + Variable, VariableOp, Wide, Width, +}; +use core::fmt::{self, Formatter}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + match *instruction { + Instruction::Movz(w) => movz(f, w), + Instruction::Movn(w) => movn(f, w), + Instruction::Movk(w) => { + f.write_str("movk ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{:#x}", w.immediate)?; + lsl_suffix(f, w.shift) + } + Instruction::ArithImm(a) => arith_imm(f, a), + Instruction::ArithReg(a) => arith_reg(f, a), + Instruction::ArithExt(a) => arith_ext(f, a), + Instruction::LogicReg(l) => logic_reg(f, l), + Instruction::LogicImm(l) => logic_imm(f, l), + Instruction::Mul(m) => mul(f, m), + Instruction::MulLong(m) => mul_long(f, m), + Instruction::MulHigh(m) => mul_high(f, m), + Instruction::Adr(a) => adr(f, cx, a), + Instruction::Csel(s) => csel(f, s), + Instruction::Variable(v) => variable(f, v), + Instruction::CondCompare(c) => cond_compare(f, c), + Instruction::ArithCarry(a) => arith_carry(f, a), + Instruction::Unary(u) => unary(f, u), + Instruction::Bitfield(b) => bitfield(f, b), + Instruction::Extract(e) => extract(f, e), + _ => unreachable!("{instruction:?} is not an integer instruction"), + } +} + +/// `, lsl #n` unless `n` is zero. +fn lsl_suffix(f: &mut Formatter<'_>, amount: u8) -> fmt::Result { + if amount == 0 { + Ok(()) + } else { + write!(f, ", lsl #{amount}") + } +} + +fn shift_name(shift: Shift) -> &'static str { + match shift { + Shift::Lsl => "lsl", + Shift::Lsr => "lsr", + Shift::Asr => "asr", + Shift::Ror => "ror", + } +} + +fn extend_name(extend: Extend) -> &'static str { + match extend { + Extend::Uxtb => "uxtb", + Extend::Uxth => "uxth", + Extend::Uxtw => "uxtw", + Extend::Uxtx => "uxtx", + Extend::Sxtb => "sxtb", + Extend::Sxth => "sxth", + Extend::Sxtw => "sxtw", + Extend::Sxtx => "sxtx", + } +} + +/// `, #n`; a zero `lsl` is not printed (a zero `ror` is). +fn shift_operand(f: &mut Formatter<'_>, shift: Shift, amount: u8) -> fmt::Result { + if shift == Shift::Lsl && amount == 0 { + Ok(()) + } else { + write!(f, ", {} #{amount}", shift_name(shift)) + } +} + +fn invert(cond: Condition) -> Condition { + use Condition::*; + match cond { + Eq => Ne, + Ne => Eq, + Cs => Cc, + Cc => Cs, + Mi => Pl, + Pl => Mi, + Vs => Vc, + Vc => Vs, + Hi => Ls, + Ls => Hi, + Ge => Lt, + Lt => Ge, + Gt => Le, + Le => Gt, + Al => Nv, + Nv => Al, + } +} + +fn bits(width: Width) -> u16 { + u16::from(width as u8) +} + +/// The width's all-ones mask. +fn mask(width: Width) -> u64 { + match width { + Width::W32 => u64::from(u32::MAX), + Width::X64 => u64::MAX, + } +} + +fn movz(f: &mut Formatter<'_>, w: Wide) -> fmt::Result { + if w.immediate == 0 && w.shift != 0 { + f.write_str("movz ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{:#x}", w.immediate)?; + return lsl_suffix(f, w.shift); + } + let value = (u64::from(w.immediate) << (w.shift & 63)) & mask(w.width); + f.write_str("mov ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{value:#x}") +} + +fn movn(f: &mut Formatter<'_>, w: Wide) -> fmt::Result { + // `mov` only where a `movz` could not have produced the same value. + if (w.immediate == 0 && w.shift != 0) || (w.width == Width::W32 && w.immediate == 0xffff) { + f.write_str("movn ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{:#x}", w.immediate)?; + return lsl_suffix(f, w.shift); + } + let value = !(u64::from(w.immediate) << (w.shift & 63)) & mask(w.width); + f.write_str("mov ")?; + gpr(f, w.width, w.rd)?; + write!(f, ", #{value:#x}") +} + +fn arith_imm(f: &mut Formatter<'_>, a: Arithmetic) -> fmt::Result { + let add = a.op == ArithOp::Add; + if !a.flags && add && a.immediate == 0 && !a.shifted && (a.rd == 31 || a.rn == 31) { + f.write_str("mov ")?; + gpr_sp(f, a.width, a.rd)?; + f.write_str(", ")?; + return gpr_sp(f, a.width, a.rn); + } + if a.flags && a.rd == 31 { + f.write_str(if add { "cmn " } else { "cmp " })?; + } else { + f.write_str(match (add, a.flags) { + (true, false) => "add ", + (true, true) => "adds ", + (false, false) => "sub ", + (false, true) => "subs ", + })?; + if a.flags { + gpr(f, a.width, a.rd)?; + } else { + gpr_sp(f, a.width, a.rd)?; + } + f.write_str(", ")?; + } + gpr_sp(f, a.width, a.rn)?; + write!(f, ", #{:#x}", a.immediate)?; + if a.shifted { + f.write_str(", lsl #12")?; + } + Ok(()) +} + +fn arith_reg(f: &mut Formatter<'_>, a: ArithmeticReg) -> fmt::Result { + let add = a.op == ArithOp::Add; + let mut rn = Some(a.rn); + if a.flags && a.rd == 31 { + f.write_str(if add { "cmn " } else { "cmp " })?; + } else if !add && a.rn == 31 { + f.write_str(if a.flags { "negs " } else { "neg " })?; + gpr(f, a.width, a.rd)?; + f.write_str(", ")?; + rn = None; + } else { + f.write_str(match (add, a.flags) { + (true, false) => "add ", + (true, true) => "adds ", + (false, false) => "sub ", + (false, true) => "subs ", + })?; + gpr(f, a.width, a.rd)?; + f.write_str(", ")?; + } + if let Some(rn) = rn { + gpr(f, a.width, rn)?; + f.write_str(", ")?; + } + gpr(f, a.width, a.rm)?; + shift_operand(f, a.shift, a.amount) +} + +fn arith_ext(f: &mut Formatter<'_>, a: ArithmeticExtended) -> fmt::Result { + let add = a.op == ArithOp::Add; + // `rd` is the stack pointer only where the instruction does not set flags. + let rd_sp = !a.flags && a.rd == 31; + if a.flags && a.rd == 31 { + f.write_str(if add { "cmn " } else { "cmp " })?; + } else { + f.write_str(match (add, a.flags) { + (true, false) => "add ", + (true, true) => "adds ", + (false, false) => "sub ", + (false, true) => "subs ", + })?; + if a.flags { + gpr(f, a.width, a.rd)?; + } else { + gpr_sp(f, a.width, a.rd)?; + } + f.write_str(", ")?; + } + gpr_sp(f, a.width, a.rn)?; + f.write_str(", ")?; + let wide_rm = a.width == Width::X64 && matches!(a.extend, Extend::Uxtx | Extend::Sxtx); + gpr(f, if wide_rm { Width::X64 } else { Width::W32 }, a.rm)?; + let native = match a.width { + Width::W32 => Extend::Uxtw, + Width::X64 => Extend::Uxtx, + }; + if (rd_sp || a.rn == 31) && a.extend == native { + lsl_suffix(f, a.amount) + } else { + write!(f, ", {}", extend_name(a.extend))?; + if a.amount != 0 { + write!(f, " #{}", a.amount)?; + } + Ok(()) + } +} + +fn logic_reg(f: &mut Formatter<'_>, l: Logic) -> fmt::Result { + let plain_shift = l.shift == Shift::Lsl && l.amount == 0; + if l.op == LogicOp::BitOr && !l.flags && l.rn == 31 { + if !l.invert && plain_shift { + f.write_str("mov ")?; + gpr(f, l.width, l.rd)?; + f.write_str(", ")?; + return gpr(f, l.width, l.rm); + } + if l.invert { + f.write_str("mvn ")?; + gpr(f, l.width, l.rd)?; + f.write_str(", ")?; + gpr(f, l.width, l.rm)?; + return shift_operand(f, l.shift, l.amount); + } + } + if l.op == LogicOp::BitAnd && l.flags && !l.invert && l.rd == 31 { + f.write_str("tst ")?; + } else { + f.write_str(match (l.op, l.invert, l.flags) { + (LogicOp::BitAnd, false, false) => "and ", + (LogicOp::BitAnd, false, true) => "ands ", + (LogicOp::BitAnd, true, false) => "bic ", + (LogicOp::BitAnd, true, true) => "bics ", + (LogicOp::BitOr, false, _) => "orr ", + (LogicOp::BitOr, true, _) => "orn ", + (LogicOp::BitXor, false, _) => "eor ", + (LogicOp::BitXor, true, _) => "eon ", + })?; + gpr(f, l.width, l.rd)?; + f.write_str(", ")?; + } + gpr(f, l.width, l.rn)?; + f.write_str(", ")?; + gpr(f, l.width, l.rm)?; + shift_operand(f, l.shift, l.amount) +} + +/// Whether a single `movz` or `movn` can load `value` (already masked to `width`). +fn wide_constant(value: u64, width: Width) -> bool { + let single_chunk = + |v: u64| v != 0 && [0, 16, 32, 48].iter().any(|&s| v & !(0xffffu64 << s) == 0); + single_chunk(value) || single_chunk(!value & mask(width)) +} + +fn logic_imm(f: &mut Formatter<'_>, l: LogicImm) -> fmt::Result { + let value = l.immediate & mask(l.width); + if l.op == LogicOp::BitOr + && !l.flags + && l.rn == 31 + && (l.rd == 31 || !wide_constant(value, l.width)) + { + f.write_str("mov ")?; + gpr_sp(f, l.width, l.rd)?; + return write!(f, ", #{value:#x}"); + } + if l.op == LogicOp::BitAnd && l.flags && l.rd == 31 { + f.write_str("tst ")?; + } else { + f.write_str(match (l.op, l.flags) { + (LogicOp::BitAnd, false) => "and ", + (LogicOp::BitAnd, true) => "ands ", + (LogicOp::BitOr, _) => "orr ", + (LogicOp::BitXor, _) => "eor ", + })?; + if l.flags { + gpr(f, l.width, l.rd)?; + } else { + gpr_sp(f, l.width, l.rd)?; + } + f.write_str(", ")?; + } + gpr(f, l.width, l.rn)?; + write!(f, ", #{value:#x}") +} + +fn mul(f: &mut Formatter<'_>, m: Multiply) -> fmt::Result { + let add = m.op == ArithOp::Add; + let alias = m.ra == 31; + f.write_str(match (add, alias) { + (true, true) => "mul ", + (false, true) => "mneg ", + (true, false) => "madd ", + (false, false) => "msub ", + })?; + gpr(f, m.width, m.rd)?; + f.write_str(", ")?; + gpr(f, m.width, m.rn)?; + f.write_str(", ")?; + gpr(f, m.width, m.rm)?; + if !alias { + f.write_str(", ")?; + gpr(f, m.width, m.ra)?; + } + Ok(()) +} + +fn mul_long(f: &mut Formatter<'_>, m: MultiplyLong) -> fmt::Result { + let add = m.op == ArithOp::Add; + let alias = m.ra == 31; + f.write_str(match (m.signed, add, alias) { + (true, true, true) => "smull ", + (true, false, true) => "smnegl ", + (true, true, false) => "smaddl ", + (true, false, false) => "smsubl ", + (false, true, true) => "umull ", + (false, false, true) => "umnegl ", + (false, true, false) => "umaddl ", + (false, false, false) => "umsubl ", + })?; + gpr(f, Width::X64, m.rd)?; + f.write_str(", ")?; + gpr(f, Width::W32, m.rn)?; + f.write_str(", ")?; + gpr(f, Width::W32, m.rm)?; + if !alias { + f.write_str(", ")?; + gpr(f, Width::X64, m.ra)?; + } + Ok(()) +} + +fn mul_high(f: &mut Formatter<'_>, m: MultiplyHigh) -> fmt::Result { + f.write_str(if m.signed { "smulh " } else { "umulh " })?; + gpr(f, Width::X64, m.rd)?; + f.write_str(", ")?; + gpr(f, Width::X64, m.rn)?; + f.write_str(", ")?; + gpr(f, Width::X64, m.rm) +} + +fn adr(f: &mut Formatter<'_>, cx: &Context<'_>, a: Address) -> fmt::Result { + f.write_str(if a.page { "adrp " } else { "adr " })?; + gpr(f, Width::X64, a.rd)?; + f.write_str(", ")?; + if a.page { + cx.page_target(f, a.offset) + } else { + cx.target(f, a.offset) + } +} + +fn csel(f: &mut Formatter<'_>, s: Select) -> fmt::Result { + let real = !matches!(s.cond, Condition::Al | Condition::Nv); + let same = s.rn == s.rm; + // opc: 0 csel, 1 csinc, 2 csinv, 3 csneg. + if real && same { + let alias = match s.opc { + 1 if s.rn == 31 => Some(("cset ", false)), + 1 => Some(("cinc ", true)), + 2 if s.rn == 31 => Some(("csetm ", false)), + 2 => Some(("cinv ", true)), + 3 => Some(("cneg ", true)), + _ => None, + }; + if let Some((name, with_rn)) = alias { + f.write_str(name)?; + gpr(f, s.width, s.rd)?; + f.write_str(", ")?; + if with_rn { + gpr(f, s.width, s.rn)?; + f.write_str(", ")?; + } + return f.write_str(condition(invert(s.cond))); + } + } + f.write_str(match s.opc { + 0 => "csel ", + 1 => "csinc ", + 2 => "csinv ", + _ => "csneg ", + })?; + gpr(f, s.width, s.rd)?; + f.write_str(", ")?; + gpr(f, s.width, s.rn)?; + f.write_str(", ")?; + gpr(f, s.width, s.rm)?; + write!(f, ", {}", condition(s.cond)) +} + +fn variable(f: &mut Formatter<'_>, v: Variable) -> fmt::Result { + f.write_str(match v.op { + VariableOp::Udiv => "udiv ", + VariableOp::Sdiv => "sdiv ", + VariableOp::Lsl => "lsl ", + VariableOp::Lsr => "lsr ", + VariableOp::Asr => "asr ", + VariableOp::Ror => "ror ", + })?; + gpr(f, v.width, v.rd)?; + f.write_str(", ")?; + gpr(f, v.width, v.rn)?; + f.write_str(", ")?; + gpr(f, v.width, v.rm) +} + +fn cond_compare(f: &mut Formatter<'_>, c: CondCompare) -> fmt::Result { + f.write_str(if c.op == ArithOp::Add { + "ccmn " + } else { + "ccmp " + })?; + gpr(f, c.width, c.rn)?; + f.write_str(", ")?; + match c.operand { + CompareOperand::Register(rm) => gpr(f, c.width, rm)?, + CompareOperand::Immediate(imm) => write!(f, "#{imm:#x}")?, + } + write!(f, ", #{:#x}, {}", c.nzcv, condition(c.cond)) +} + +fn arith_carry(f: &mut Formatter<'_>, a: AddCarry) -> fmt::Result { + if a.op == ArithOp::Sub && a.rn == 31 { + f.write_str(if a.flags { "ngcs " } else { "ngc " })?; + gpr(f, a.width, a.rd)?; + f.write_str(", ")?; + return gpr(f, a.width, a.rm); + } + f.write_str(match (a.op, a.flags) { + (ArithOp::Add, false) => "adc ", + (ArithOp::Add, true) => "adcs ", + (ArithOp::Sub, false) => "sbc ", + (ArithOp::Sub, true) => "sbcs ", + })?; + gpr(f, a.width, a.rd)?; + f.write_str(", ")?; + gpr(f, a.width, a.rn)?; + f.write_str(", ")?; + gpr(f, a.width, a.rm) +} + +fn unary(f: &mut Formatter<'_>, u: Unary) -> fmt::Result { + f.write_str(match u.op { + UnaryOp::Rbit => "rbit ", + UnaryOp::Rev16 => "rev16 ", + UnaryOp::Rev32 => "rev32 ", + UnaryOp::Rev => "rev ", + UnaryOp::Clz => "clz ", + })?; + gpr(f, u.width, u.rd)?; + f.write_str(", ")?; + gpr(f, u.width, u.rn) +} + +/// `name rd, rn, #a` or `name rd, rn, #a, #b`. +fn bitfield_alias( + f: &mut Formatter<'_>, + name: &str, + b: Bitfield, + first: u16, + second: Option, +) -> fmt::Result { + write!(f, "{name} ")?; + gpr(f, b.width, b.rd)?; + f.write_str(", ")?; + gpr(f, b.width, b.rn)?; + write!(f, ", #{first}")?; + if let Some(second) = second { + write!(f, ", #{second}")?; + } + Ok(()) +} + +fn bitfield(f: &mut Formatter<'_>, b: Bitfield) -> fmt::Result { + let lsb = u16::from(b.lsb); + let len = u16::from(b.len); + let top = lsb.wrapping_add(len) == bits(b.width); + match b.op { + BitfieldOp::Signed => { + if b.extract && lsb == 0 { + let ext = match (len, b.width) { + (8, _) => Some("sxtb "), + (16, _) => Some("sxth "), + (32, Width::X64) => Some("sxtw "), + _ => None, + }; + if let Some(name) = ext { + f.write_str(name)?; + gpr(f, b.width, b.rd)?; + f.write_str(", ")?; + return gpr(f, Width::W32, b.rn); + } + } + if b.extract && top { + bitfield_alias(f, "asr", b, lsb, None) + } else if b.extract { + bitfield_alias(f, "sbfx", b, lsb, Some(len)) + } else { + bitfield_alias(f, "sbfiz", b, lsb, Some(len)) + } + } + BitfieldOp::Unsigned => { + if b.extract && lsb == 0 && b.width == Width::W32 && (len == 8 || len == 16) { + f.write_str(if len == 8 { "uxtb " } else { "uxth " })?; + gpr(f, b.width, b.rd)?; + f.write_str(", ")?; + return gpr(f, b.width, b.rn); + } + if !b.extract && top { + bitfield_alias(f, "lsl", b, lsb, None) + } else if b.extract && top { + bitfield_alias(f, "lsr", b, lsb, None) + } else if b.extract { + bitfield_alias(f, "ubfx", b, lsb, Some(len)) + } else { + bitfield_alias(f, "ubfiz", b, lsb, Some(len)) + } + } + BitfieldOp::Insert => { + if !b.extract && b.rn == 31 { + write!(f, "bfc ")?; + gpr(f, b.width, b.rd)?; + write!(f, ", #{lsb}, #{len}") + } else if b.extract { + bitfield_alias(f, "bfxil", b, lsb, Some(len)) + } else { + bitfield_alias(f, "bfi", b, lsb, Some(len)) + } + } + } +} + +fn extract(f: &mut Formatter<'_>, e: Extract) -> fmt::Result { + let ror = e.rn == e.rm; + f.write_str(if ror { "ror " } else { "extr " })?; + gpr(f, e.width, e.rd)?; + f.write_str(", ")?; + gpr(f, e.width, e.rn)?; + if !ror { + f.write_str(", ")?; + gpr(f, e.width, e.rm)?; + } + write!(f, ", #{}", e.lsb) +} + +#[cfg(test)] +mod tests { + extern crate std; + use super::*; + use crate::decode::decode; + use core::fmt::Write as _; + use std::string::String; + + struct At<'a>(&'a Instruction, Option); + + impl core::fmt::Display for At<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let cx = Context { + pc: self.1, + symbols: &(), + }; + write(f, &cx, self.0) + } + } + + fn text(word: u32, pc: Option) -> String { + let instruction = decode(word).expect("decodes"); + let mut out = String::new(); + write!(out, "{}", At(&instruction, pc)).unwrap(); + out + } + + /// Words and their GNU objdump rendering. + const CASES: &[(u32, &str)] = &[ + (0xd2800000, "mov x0, #0x0"), + (0xd2a00000, "movz x0, #0x0, lsl #16"), + (0x52a00000, "movz w0, #0x0, lsl #16"), + (0xd2e000a0, "mov x0, #0x5000000000000"), + (0x92800000, "mov x0, #0xffffffffffffffff"), + (0x12800000, "mov w0, #0xffffffff"), + (0x92a00000, "movn x0, #0x0, lsl #16"), + (0x12a00000, "movn w0, #0x0, lsl #16"), + (0x129fffe0, "movn w0, #0xffff"), + (0x12bfffe0, "movn w0, #0xffff, lsl #16"), + (0x929fffe0, "mov x0, #0xffffffffffff0000"), + (0x12a000a0, "mov w0, #0xfffaffff"), + (0xf28000a0, "movk x0, #0x5"), + (0x72a000a0, "movk w0, #0x5, lsl #16"), + (0xf2ffffe0, "movk x0, #0xffff, lsl #48"), + (0x91400420, "add x0, x1, #0x1, lsl #12"), + (0x917ffc20, "add x0, x1, #0xfff, lsl #12"), + (0x91400020, "add x0, x1, #0x0, lsl #12"), + (0x9140003f, "add sp, x1, #0x0, lsl #12"), + (0x914003e0, "add x0, sp, #0x0, lsl #12"), + (0xd140003f, "sub sp, x1, #0x0, lsl #12"), + (0xf140003f, "cmp x1, #0x0, lsl #12"), + (0xb140003f, "cmn x1, #0x0, lsl #12"), + (0x11400c20, "add w0, w1, #0x3, lsl #12"), + (0x7140001f, "cmp w0, #0x0, lsl #12"), + (0x9100003f, "mov sp, x1"), + (0x910003e0, "mov x0, sp"), + (0x91000020, "add x0, x1, #0x0"), + (0xb10003e0, "adds x0, sp, #0x0"), + (0xb1000c3f, "cmn x1, #0x3"), + (0x71000fff, "cmp wsp, #0x3"), + (0xf1000fe0, "subs x0, sp, #0x3"), + (0xd10043ff, "sub sp, sp, #0x10"), + (0xcb0103e0, "neg x0, x1"), + (0xeb010fe0, "negs x0, x1, lsl #3"), + (0xeb0103ff, "cmp xzr, x1"), + (0xab82103f, "cmn x1, x2, asr #4"), + (0xab0203ff, "cmn xzr, x2"), + (0x4b810fe0, "neg w0, w1, asr #3"), + (0x8b020020, "add x0, x1, x2"), + (0x8b224020, "add x0, x1, w2, uxtw"), + (0x8b224820, "add x0, x1, w2, uxtw #2"), + (0x8b226020, "add x0, x1, x2, uxtx"), + (0x8b226c20, "add x0, x1, x2, uxtx #3"), + (0x8b22e420, "add x0, x1, x2, sxtx #1"), + (0x8b22d020, "add x0, x1, w2, sxtw #4"), + (0x8b2263ff, "add sp, sp, x2"), + (0x8b226bff, "add sp, sp, x2, lsl #2"), + (0x8b226c3f, "add sp, x1, x2, lsl #3"), + (0x8b226fe0, "add x0, sp, x2, lsl #3"), + (0x0b2247e0, "add w0, wsp, w2, lsl #1"), + (0x0b2243e0, "add w0, wsp, w2"), + (0x0b226020, "add w0, w1, w2, uxtx"), + (0x0b224020, "add w0, w1, w2, uxtw"), + (0x0b228820, "add w0, w1, w2, sxtb #2"), + (0xab2267e0, "adds x0, sp, x2, lsl #1"), + (0xab2263e0, "adds x0, sp, x2"), + (0xab2267ff, "cmn sp, x2, lsl #1"), + (0xab22643f, "cmn x1, x2, uxtx #1"), + (0xeb2203ff, "cmp sp, w2, uxtb"), + (0x6b22a7ff, "cmp wsp, w2, sxth #1"), + (0xab22c03f, "cmn x1, w2, sxtw"), + (0xcb22703f, "sub sp, x1, x2, lsl #4"), + (0xcb228020, "sub x0, x1, w2, sxtb"), + (0x129fffc0, "mov w0, #0xffff0001"), + (0x12bfffc0, "mov w0, #0x1ffff"), + (0x92bfffe0, "mov x0, #0xffffffff0000ffff"), + (0x92dfffe0, "mov x0, #0xffff0000ffffffff"), + (0x92ffffe0, "mov x0, #0xffffffffffff"), + (0x92e00020, "mov x0, #0xfffeffffffffffff"), + (0x52bfffe0, "mov w0, #0xffff0000"), + (0x529fffe0, "mov w0, #0xffff"), + (0xd2ffffe0, "mov x0, #0xffff000000000000"), + (0x8a020020, "and x0, x1, x2"), + (0x8a020c20, "and x0, x1, x2, lsl #3"), + (0x8ac20020, "and x0, x1, x2, ror #0"), + (0x8ac20c20, "and x0, x1, x2, ror #3"), + (0xaa0203e0, "mov x0, x2"), + (0xaa020fe0, "orr x0, xzr, x2, lsl #3"), + (0xaa1f0020, "orr x0, x1, xzr"), + (0x2a0203e0, "mov w0, w2"), + (0xaa1f03e0, "mov x0, xzr"), + (0xaa2203e0, "mvn x0, x2"), + (0xaa6213e0, "mvn x0, x2, lsr #4"), + (0xaa220020, "orn x0, x1, x2"), + (0xca2203e0, "eon x0, xzr, x2"), + (0xca0203e0, "eor x0, xzr, x2"), + (0x8a220020, "bic x0, x1, x2"), + (0xea220020, "bics x0, x1, x2"), + (0xea22003f, "bics xzr, x1, x2"), + (0xea02003f, "tst x1, x2"), + (0xea02143f, "tst x1, x2, lsl #5"), + (0xea0203ff, "tst xzr, x2"), + (0xea020020, "ands x0, x1, x2"), + (0xaae203e0, "mvn x0, x2, ror #0"), + (0xaac203e0, "orr x0, xzr, x2, ror #0"), + (0xaa8207e0, "orr x0, xzr, x2, asr #1"), + (0x92401c20, "and x0, x1, #0xff"), + (0x92401c3f, "and sp, x1, #0xff"), + (0xb2401fe0, "orr x0, xzr, #0xff"), + (0xb200f3e0, "mov x0, #0x5555555555555555"), + (0x32089fe0, "mov w0, #0xff00ff00"), + (0x320003e0, "orr w0, wzr, #0x1"), + (0x320003ff, "mov wsp, #0x1"), + (0xb24003ff, "mov sp, #0x1"), + (0xb278dfe0, "orr x0, xzr, #0xffffffffffffff00"), + (0xb2410020, "orr x0, x1, #0x8000000000000000"), + (0x321f7820, "orr w0, w1, #0xfffffffe"), + (0xf27c0c3f, "tst x1, #0xf0"), + (0xf27c0c20, "ands x0, x1, #0xf0"), + (0x721c0c3f, "tst w1, #0xf0"), + (0xd2401fe0, "eor x0, xzr, #0xff"), + (0x52007820, "eor w0, w1, #0x7fffffff"), + (0xb27f7be0, "mov x0, #0xfffffffe"), + (0xb240fbe0, "orr x0, xzr, #0x7fffffffffffffff"), + (0x321f7be0, "orr w0, wzr, #0xfffffffe"), + (0x9b027c20, "mul x0, x1, x2"), + (0x9b020c20, "madd x0, x1, x2, x3"), + (0x9b02fc20, "mneg x0, x1, x2"), + (0x9b028c20, "msub x0, x1, x2, x3"), + (0x1b02fc20, "mneg w0, w1, w2"), + (0x1b027c20, "mul w0, w1, w2"), + (0x9b220c20, "smaddl x0, w1, w2, x3"), + (0x9b227c20, "smull x0, w1, w2"), + (0x9b228c20, "smsubl x0, w1, w2, x3"), + (0x9b22fc20, "smnegl x0, w1, w2"), + (0x9ba20c20, "umaddl x0, w1, w2, x3"), + (0x9ba27c20, "umull x0, w1, w2"), + (0x9ba28c20, "umsubl x0, w1, w2, x3"), + (0x9ba2fc20, "umnegl x0, w1, w2"), + (0x9b427c20, "smulh x0, x1, x2"), + (0x9bc27c20, "umulh x0, x1, x2"), + (0x9a820020, "csel x0, x1, x2, eq"), + (0x9a82e020, "csel x0, x1, x2, al"), + (0x9a82f020, "csel x0, x1, x2, nv"), + (0x9a821420, "csinc x0, x1, x2, ne"), + (0x9a811420, "cinc x0, x1, eq"), + (0x9a81e420, "csinc x0, x1, x1, al"), + (0x9a81f420, "csinc x0, x1, x1, nv"), + (0x9a9f17e0, "cset x0, eq"), + (0x9a9fe7e0, "csinc x0, xzr, xzr, al"), + (0x9a9ff7e0, "csinc x0, xzr, xzr, nv"), + (0x9a8117e0, "csinc x0, xzr, x1, ne"), + (0x9a9f1420, "csinc x0, x1, xzr, ne"), + (0xda9f13e0, "csetm x0, eq"), + (0xda9fe3e0, "csinv x0, xzr, xzr, al"), + (0xda811020, "cinv x0, x1, eq"), + (0xda81f020, "csinv x0, x1, x1, nv"), + (0xda8113e0, "csinv x0, xzr, x1, ne"), + (0xda811420, "cneg x0, x1, eq"), + (0xda81e420, "csneg x0, x1, x1, al"), + (0xda9f17e0, "cneg x0, xzr, eq"), + (0xda9fe7e0, "csneg x0, xzr, xzr, al"), + (0xda821420, "csneg x0, x1, x2, ne"), + (0x1a9f27e0, "cset w0, cc"), + (0x1a9f87e0, "cset w0, ls"), + (0x1a9f47e0, "cset w0, pl"), + (0x1a9f77e0, "cset w0, vs"), + (0x1a9fa7e0, "cset w0, lt"), + (0x1a9fd7e0, "cset w0, gt"), + (0x9ac20820, "udiv x0, x1, x2"), + (0x1ac20c20, "sdiv w0, w1, w2"), + (0x9ac22020, "lsl x0, x1, x2"), + (0x1ac22420, "lsr w0, w1, w2"), + (0x9ac22820, "asr x0, x1, x2"), + (0x1ac22c20, "ror w0, w1, w2"), + (0xfa410005, "ccmp x0, x1, #0x5, eq"), + (0xfa5f180f, "ccmp x0, #0x1f, #0xf, ne"), + (0x3a41e000, "ccmn w0, w1, #0x0, al"), + (0x3a40f800, "ccmn w0, #0x0, #0x0, nv"), + (0xfa5f23e1, "ccmp xzr, xzr, #0x1, cs"), + (0x9a020020, "adc x0, x1, x2"), + (0x3a020020, "adcs w0, w1, w2"), + (0xda020020, "sbc x0, x1, x2"), + (0xda0203e0, "ngc x0, x2"), + (0xfa0203e0, "ngcs x0, x2"), + (0x7a0203e0, "ngcs w0, w2"), + (0x5a1f03e0, "ngc w0, wzr"), + (0x9a1f03e0, "adc x0, xzr, xzr"), + (0xdac00020, "rbit x0, x1"), + (0x5ac00020, "rbit w0, w1"), + (0xdac00420, "rev16 x0, x1"), + (0x5ac00420, "rev16 w0, w1"), + (0xdac00820, "rev32 x0, x1"), + (0x5ac00820, "rev w0, w1"), + (0xdac00c20, "rev x0, x1"), + (0xdac01020, "clz x0, x1"), + (0x5ac013e0, "clz w0, wzr"), + (0x93401c20, "sxtb x0, w1"), + (0x93403c20, "sxth x0, w1"), + (0x93407c20, "sxtw x0, w1"), + (0x13001c20, "sxtb w0, w1"), + (0x13003c20, "sxth w0, w1"), + (0x13007c20, "asr w0, w1, #0"), + (0x9340fc20, "asr x0, x1, #0"), + (0x9345fc20, "asr x0, x1, #5"), + (0x13057c20, "asr w0, w1, #5"), + (0x93452420, "sbfx x0, x1, #5, #5"), + (0x93491420, "sbfiz x0, x1, #55, #6"), + (0x937ff820, "sbfiz x0, x1, #1, #63"), + (0x131f7820, "sbfiz w0, w1, #1, #31"), + (0x93410020, "sbfiz x0, x1, #63, #1"), + (0x93400020, "sbfx x0, x1, #0, #1"), + (0x93402020, "sbfx x0, x1, #0, #9"), + (0x93430c20, "sbfx x0, x1, #3, #1"), + (0xd3401c20, "ubfx x0, x1, #0, #8"), + (0x53001c20, "uxtb w0, w1"), + (0x53003c20, "uxth w0, w1"), + (0xd3403c20, "ubfx x0, x1, #0, #16"), + (0xd3407c20, "ubfx x0, x1, #0, #32"), + (0xd340fc20, "lsr x0, x1, #0"), + (0x53007c20, "lsr w0, w1, #0"), + (0xd345fc20, "lsr x0, x1, #5"), + (0x53057c20, "lsr w0, w1, #5"), + (0xd3452420, "ubfx x0, x1, #5, #5"), + (0xd3491420, "ubfiz x0, x1, #55, #6"), + (0xd37ff820, "lsl x0, x1, #1"), + (0x531f7820, "lsl w0, w1, #1"), + (0xd3410020, "lsl x0, x1, #63"), + (0xd3400020, "ubfx x0, x1, #0, #1"), + (0x53010020, "lsl w0, w1, #31"), + (0xd3430c20, "ubfx x0, x1, #3, #1"), + (0xd340f820, "ubfx x0, x1, #0, #63"), + (0xd37df020, "lsl x0, x1, #3"), + (0xb3452420, "bfxil x0, x1, #5, #5"), + (0xb3491420, "bfi x0, x1, #55, #6"), + (0xb34917e0, "bfc x0, #55, #6"), + (0xb34527e0, "bfxil x0, xzr, #5, #5"), + (0xb34027e0, "bfxil x0, xzr, #0, #10"), + (0xb3400020, "bfxil x0, x1, #0, #1"), + (0xb37ff820, "bfi x0, x1, #1, #63"), + (0x330103e0, "bfc w0, #31, #1"), + (0x33007c20, "bfxil w0, w1, #0, #32"), + (0xb340fc20, "bfxil x0, x1, #0, #64"), + (0xb340ffe0, "bfxil x0, xzr, #0, #64"), + (0x93c21420, "extr x0, x1, x2, #5"), + (0x93c11420, "ror x0, x1, #5"), + (0x13810020, "ror w0, w1, #0"), + (0x93c20020, "extr x0, x1, x2, #0"), + (0x13827c20, "extr w0, w1, w2, #31"), + (0x93dfffe0, "ror x0, xzr, #63"), + ]; + + #[test] + fn matches_objdump() { + for &(word, expected) in CASES { + assert_eq!(text(word, None), expected, "{word:#010x}"); + } + } + + #[test] + fn pc_relative() { + assert_eq!(text(0x1000_0020, None), "adr x0, .+4"); + assert_eq!(text(0x10ff_ffe0, None), "adr x0, .-4"); + assert_eq!(text(0xb000_0000, None), "adrp x0, .+4096"); + assert_eq!(text(0x90ff_ffe0, None), "adrp x0, .-16384"); + assert_eq!(text(0x1000_0020, Some(0x1000)), "adr x0, 0x1004"); + assert_eq!(text(0xb000_0000, Some(0x1234)), "adrp x0, 0x2000"); + assert_eq!(text(0xb000_001f, Some(0x1234)), "adrp xzr, 0x2000"); + } +} diff --git a/src/isa/aarch64/src/format/memory.rs b/src/isa/aarch64/src/format/memory.rs new file mode 100644 index 0000000..1ee15c0 --- /dev/null +++ b/src/isa/aarch64/src/format/memory.rs @@ -0,0 +1,870 @@ +//! Loads and stores: scalar, pair, literal, exclusive, LSE atomics, prefetch and the +//! SIMD&FP and structure forms. +//! +//! Owns `Memory`, `Pair`, `Literal`, `SimdLiteral`, `Exclusive`, `Atomic`, `Prefetch`, +//! `PrefetchLiteral`, `SimdPair`, `SimdMemory` and `SimdStruct`. +//! +//! The decoded forms keep what distinguishes the instructions: `Memory::ordered` for +//! `ldar`/`stlr`/`ldapr`/`ldlar`/`stllr`, `MemoryOrder` for the `a`/`l` of exclusives and +//! atomics, `Addressing::Unscaled` for `ldur`/`prfum`, `Pair::non_temporal` for `ldnp` and +//! `SimdAccess::Ld1` for a one-register `ld1`/`st1`. + +use super::Context; +use super::reg::{fp_scalar, gpr, gpr_sp, vector}; +use crate::decode::{ + Addressing, Atomic, AtomicKind, Exclusive, Extend, Instruction, Literal, Memory, MemoryOp, + MemoryOrder, Ordered, Pair, Prefetch, PrefetchLiteral, RangePrefetch, SignExtend, SimdAccess, + SimdLiteral, SimdMemory, SimdPair, SimdStruct, Size, StructPost, Width, +}; +use crate::simd_struct::{Shape, StructDesc}; +use core::fmt::{self, Formatter}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + match instruction { + Instruction::Memory(i) => memory(f, i), + Instruction::Pair(i) => pair(f, i), + Instruction::Literal(i) => literal(f, cx, i), + Instruction::SimdLiteral(i) => simd_literal(f, cx, i), + Instruction::Exclusive(i) => exclusive(f, i), + Instruction::Atomic(i) => atomic(f, i), + Instruction::Prefetch(i) => prefetch(f, i), + Instruction::PrefetchLiteral(i) => prefetch_literal(f, cx, i), + Instruction::RangePrefetch(i) => range_prefetch(f, i), + Instruction::SimdPair(i) => simd_pair(f, i), + Instruction::SimdMemory(i) => simd_memory(f, i), + Instruction::SimdStruct(i) => simd_struct(f, i), + _ => unreachable!("{instruction:?} is not a memory instruction"), + } +} + +fn width_of(size: Size) -> Width { + if size == Size::Double { + Width::X64 + } else { + Width::W32 + } +} + +/// `[xn]`, `[xn, #imm]`, `[xn, #imm]!`, `[xn], #imm` or `[xn, rm{, ext{ #n}}]`. +fn address(f: &mut Formatter<'_>, rn: u8, addressing: Addressing) -> fmt::Result { + f.write_str("[")?; + gpr_sp(f, Width::X64, rn)?; + match addressing { + Addressing::Offset(0) | Addressing::Unscaled(0) | Addressing::Unprivileged(0) => { + f.write_str("]") + } + Addressing::Offset(offset) + | Addressing::Unscaled(offset) + | Addressing::Unprivileged(offset) => write!(f, ", #{offset}]"), + Addressing::PreIndex(offset) => write!(f, ", #{offset}]!"), + Addressing::PostIndex(offset) => write!(f, "], #{offset}"), + Addressing::Register { rm, extend, amount } => { + f.write_str(", ")?; + let wide = matches!(extend, Extend::Uxtx | Extend::Sxtx); + gpr(f, if wide { Width::X64 } else { Width::W32 }, rm)?; + let name = match (extend, amount) { + (Extend::Uxtx, None) => return f.write_str("]"), + (Extend::Uxtx, Some(_)) => "lsl", + (Extend::Uxtb, _) => "uxtb", + (Extend::Uxth, _) => "uxth", + (Extend::Uxtw, _) => "uxtw", + (Extend::Sxtb, _) => "sxtb", + (Extend::Sxth, _) => "sxth", + (Extend::Sxtw, _) => "sxtw", + (Extend::Sxtx, _) => "sxtx", + }; + write!(f, ", {name}")?; + if let Some(amount) = amount { + write!(f, " #{amount}")?; + } + f.write_str("]") + } + } +} + +fn memory(f: &mut Formatter<'_>, i: &Memory) -> fmt::Result { + let (op, suffix) = match (i.op, i.size, i.signed) { + (MemoryOp::Load, Size::Byte, SignExtend::None) => ("ld", "b"), + (MemoryOp::Load, Size::Half, SignExtend::None) => ("ld", "h"), + (MemoryOp::Load, Size::Byte, _) => ("ld", "sb"), + (MemoryOp::Load, Size::Half, _) => ("ld", "sh"), + (MemoryOp::Load, Size::Word, SignExtend::None) => ("ld", ""), + (MemoryOp::Load, Size::Word, _) => ("ld", "sw"), + (MemoryOp::Load, Size::Double, _) => ("ld", ""), + (MemoryOp::Store, Size::Byte, _) => ("st", "b"), + (MemoryOp::Store, Size::Half, _) => ("st", "h"), + (MemoryOp::Store, _, _) => ("st", ""), + }; + // The middle of the mnemonic: `ldr`, `ldur`, `ldtr`, `ldar`, `ldapr`, `ldapur`, + // `ldlar`, `stlr`, `stlur`, `stllr`. + let unscaled = matches!(i.addressing, Addressing::Unscaled(_)); + let stem = match i.ordered { + None => match i.addressing { + Addressing::Unscaled(_) => "ur", + Addressing::Unprivileged(_) => "tr", + _ => "r", + }, + Some(Ordered::Acquire) => "ar", + Some(Ordered::AcquirePc) if unscaled => "apur", + Some(Ordered::AcquirePc) => "apr", + Some(Ordered::LimitedAcquire) => "lar", + Some(Ordered::Release) if unscaled => "lur", + Some(Ordered::Release) => "lr", + Some(Ordered::LimitedRelease) => "llr", + }; + let width = if i.signed == SignExtend::To64 { + Width::X64 + } else if i.signed == SignExtend::To32 { + Width::W32 + } else { + width_of(i.size) + }; + write!(f, "{op}{stem}{suffix} ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn pair(f: &mut Formatter<'_>, i: &Pair) -> fmt::Result { + let op = match (i.op, i.signed, i.non_temporal) { + (MemoryOp::Load, SignExtend::None, false) => "ldp", + (MemoryOp::Load, SignExtend::None, true) => "ldnp", + (MemoryOp::Load, _, _) => "ldpsw", + (MemoryOp::Store, _, false) => "stp", + (MemoryOp::Store, _, true) => "stnp", + }; + let width = if i.signed != SignExtend::None { + Width::X64 + } else { + width_of(i.size) + }; + write!(f, "{op} ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + gpr(f, width, i.rt2)?; + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn literal(f: &mut Formatter<'_>, cx: &Context<'_>, i: &Literal) -> fmt::Result { + if i.signed == SignExtend::None { + f.write_str("ldr ")?; + gpr(f, width_of(i.size), i.rt)?; + } else { + f.write_str("ldrsw ")?; + gpr(f, Width::X64, i.rt)?; + } + f.write_str(", ")?; + cx.target(f, i.offset) +} + +fn simd_literal(f: &mut Formatter<'_>, cx: &Context<'_>, i: &SimdLiteral) -> fmt::Result { + f.write_str("ldr ")?; + fp_scalar(f, i.bytes, i.rt)?; + f.write_str(", ")?; + cx.target(f, i.offset) +} + +fn exclusive(f: &mut Formatter<'_>, i: &Exclusive) -> fmt::Result { + let (op, suffix) = ( + match (i.op, i.order) { + (MemoryOp::Load, MemoryOrder { acquire: true, .. }) => "ldaxr", + (MemoryOp::Load, _) => "ldxr", + (MemoryOp::Store, MemoryOrder { release: true, .. }) => "stlxr", + (MemoryOp::Store, _) => "stxr", + }, + match i.size { + Size::Byte => "b", + Size::Half => "h", + _ => "", + }, + ); + write!(f, "{op}{suffix} ")?; + if i.op == MemoryOp::Store { + gpr(f, Width::W32, i.rs)?; + f.write_str(", ")?; + } + gpr(f, width_of(i.size), i.rt)?; + f.write_str(", ")?; + address(f, i.rn, Addressing::Offset(0)) +} + +fn atomic(f: &mut Formatter<'_>, i: &Atomic) -> fmt::Result { + let width = width_of(i.size); + let suffix = match i.size { + Size::Byte => "b", + Size::Half => "h", + _ => "", + }; + let a = if i.order.acquire { "a" } else { "" }; + let l = if i.order.release { "l" } else { "" }; + let op = match i.kind { + AtomicKind::Add => "add", + AtomicKind::Clr => "clr", + AtomicKind::Eor => "eor", + AtomicKind::Set => "set", + AtomicKind::SMax => "smax", + AtomicKind::SMin => "smin", + AtomicKind::UMax => "umax", + AtomicKind::UMin => "umin", + AtomicKind::Swap | AtomicKind::Cas | AtomicKind::Casp => "", + AtomicKind::LoadPair | AtomicKind::StorePair => "", + }; + let memory = Addressing::Offset(0); + match i.kind { + AtomicKind::Swap => { + write!(f, "swp{a}{l}{suffix} ")?; + gpr(f, width, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + } + AtomicKind::Cas => { + write!(f, "cas{a}{l}{suffix} ")?; + gpr(f, width, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + } + AtomicKind::Casp => { + write!(f, "casp{a}{l} ")?; + gpr(f, width, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rs.wrapping_add(1) & 31)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + gpr(f, width, i.rt.wrapping_add(1) & 31)?; + } + AtomicKind::LoadPair => { + write!(f, "ld{a}xp ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + gpr(f, width, i.rt2)?; + } + AtomicKind::StorePair => { + write!(f, "st{l}xp ")?; + gpr(f, Width::W32, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + f.write_str(", ")?; + gpr(f, width, i.rt2)?; + } + _ if i.rt == 31 && !i.order.acquire => { + // `ldadd Rs, xzr, [Xn]` is `stadd Rs, [Xn]`; with acquire the load must + // happen, so `ldadda Rs, xzr, [Xn]` keeps its spelling. + write!(f, "st{op}{l}{suffix} ")?; + gpr(f, width, i.rs)?; + } + _ => { + write!(f, "ld{op}{a}{l}{suffix} ")?; + gpr(f, width, i.rs)?; + f.write_str(", ")?; + gpr(f, width, i.rt)?; + } + } + f.write_str(", ")?; + address(f, i.rn, memory) +} + +/// The operation operand of `prfm`: `pldl1keep`.. or `#0x..` when unnamed. Operation 24 +/// is `ir` only for the scaled-offset form (`ir_form`); every other form spells it `#0x18`. +fn prefetch_operation(f: &mut Formatter<'_>, op: u8, ir_form: bool) -> fmt::Result { + if op == 24 && ir_form { + return f.write_str("ir"); + } + let kind = match op >> 3 { + 0 => "pld", + 1 => "pli", + 2 => "pst", + _ => return write!(f, "#{op:#x}"), + }; + let target = ["l1", "l2", "l3", "slc"][usize::from((op >> 1) & 3)]; + let policy = if op & 1 == 0 { "keep" } else { "strm" }; + write!(f, "{kind}{target}{policy}") +} + +fn prefetch(f: &mut Formatter<'_>, i: &Prefetch) -> fmt::Result { + let u = if matches!(i.addressing, Addressing::Unscaled(_)) { + "u" + } else { + "" + }; + let ir_form = matches!(i.addressing, Addressing::Offset(_)); + write!(f, "prf{u}m ")?; + prefetch_operation(f, i.op, ir_form)?; + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn prefetch_literal(f: &mut Formatter<'_>, cx: &Context<'_>, i: &PrefetchLiteral) -> fmt::Result { + f.write_str("prfm ")?; + prefetch_operation(f, i.op, false)?; + f.write_str(", ")?; + cx.target(f, i.offset) +} + +/// `rprfm pldkeep, xm, [xn]`: the four named operations, `#n` for the rest. +fn range_prefetch(f: &mut Formatter<'_>, i: &RangePrefetch) -> fmt::Result { + f.write_str("rprfm ")?; + match i.op { + 0 => f.write_str("pldkeep")?, + 1 => f.write_str("pstkeep")?, + 4 => f.write_str("pldstrm")?, + 5 => f.write_str("pststrm")?, + n => write!(f, "#{n}")?, + } + f.write_str(", ")?; + gpr(f, Width::X64, i.rm)?; + f.write_str(", ")?; + address(f, i.rn, Addressing::Offset(0)) +} + +fn simd_pair(f: &mut Formatter<'_>, i: &SimdPair) -> fmt::Result { + f.write_str(match (i.op, i.non_temporal) { + (MemoryOp::Load, false) => "ldp ", + (MemoryOp::Store, false) => "stp ", + (MemoryOp::Load, true) => "ldnp ", + (MemoryOp::Store, true) => "stnp ", + })?; + fp_scalar(f, i.bytes, i.rt)?; + f.write_str(", ")?; + fp_scalar(f, i.bytes, i.rt2)?; + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn simd_memory(f: &mut Formatter<'_>, i: &SimdMemory) -> fmt::Result { + let op = match i.op { + MemoryOp::Load => "ld", + MemoryOp::Store => "st", + }; + match i.access { + SimdAccess::Register => { + let u = if matches!(i.addressing, Addressing::Unscaled(_)) { + "u" + } else { + "" + }; + write!(f, "{op}{u}r ")?; + fp_scalar(f, i.bytes, i.rt)?; + } + SimdAccess::Ld1 { esize } => { + write!(f, "{op}1 {{")?; + vector(f, i.rt, esize.trailing_zeros() as u8, i.bytes == 16)?; + f.write_str("}")?; + } + } + f.write_str(", ")?; + address(f, i.rn, i.addressing) +} + +fn element(esize: u8) -> char { + match esize { + 1 => 'b', + 2 => 'h', + 4 => 's', + _ => 'd', + } +} + +fn register_list(f: &mut Formatter<'_>, d: &StructDesc) -> fmt::Result { + let size = (d.esize.max(1)).trailing_zeros() as u8; + let last = d.rt.wrapping_add(d.registers.wrapping_sub(1)) & 31; + f.write_str("{")?; + for (n, r) in [d.rt, last].into_iter().enumerate() { + if n == 1 { + if d.registers <= 1 { + break; + } + f.write_str("-")?; + } + match d.shape { + Shape::Lane => write!(f, "v{r}.{}", element(d.esize))?, + _ => vector(f, r, size, d.q)?, + } + } + f.write_str("}")?; + if d.shape == Shape::Lane { + write!(f, "[{}]", d.lane)?; + } + Ok(()) +} + +fn simd_struct(f: &mut Formatter<'_>, i: &SimdStruct) -> fmt::Result { + let d = &i.desc; + let count = if d.shape == Shape::Multiple && !d.interleaved { + 1 + } else { + d.registers + }; + let op = if d.store { "st" } else { "ld" }; + let replicate = if d.shape == Shape::Replicate { "r" } else { "" }; + write!(f, "{op}{count}{replicate} ")?; + register_list(f, d)?; + f.write_str(", [")?; + gpr_sp(f, Width::X64, i.rn)?; + f.write_str("]")?; + match i.post { + StructPost::None => Ok(()), + StructPost::Immediate => write!(f, ", #{}", d.bytes()), + StructPost::Register(rm) => { + f.write_str(", ")?; + gpr(f, Width::X64, rm) + } + } +} + +#[cfg(test)] +mod tests { + use crate::decode::decode; + use std::string::ToString; + + /// `(pc, word, text)`: text is what GNU objdump 2.46 prints for `word` at `pc`. + const ROWS: &[(u64, u32, &str)] = &[ + (0x0, 0xf9400020, "ldr x0, [x1]"), + (0x4, 0xf9400420, "ldr x0, [x1, #8]"), + (0x8, 0xf97ffc20, "ldr x0, [x1, #32760]"), + (0xc, 0xf94007e0, "ldr x0, [sp, #8]"), + (0x10, 0xf85f8020, "ldur x0, [x1, #-8]"), + (0x14, 0xf8401020, "ldur x0, [x1, #1]"), + (0x18, 0xb97ffc20, "ldr w0, [x1, #16380]"), + (0x1c, 0xf8408c20, "ldr x0, [x1, #8]!"), + (0x20, 0xf8500fe0, "ldr x0, [sp, #-256]!"), + (0x24, 0xf84ff420, "ldr x0, [x1], #255"), + (0x28, 0xb85fc422, "ldr w2, [x1], #-4"), + (0x2c, 0xf8626820, "ldr x0, [x1, x2]"), + (0x30, 0xf8627820, "ldr x0, [x1, x2, lsl #3]"), + (0x34, 0xf87f7820, "ldr x0, [x1, xzr, lsl #3]"), + (0x38, 0xf8624820, "ldr x0, [x1, w2, uxtw]"), + (0x3c, 0xf8625820, "ldr x0, [x1, w2, uxtw #3]"), + (0x40, 0xf862c820, "ldr x0, [x1, w2, sxtw]"), + (0x44, 0xf862d820, "ldr x0, [x1, w2, sxtw #3]"), + (0x48, 0xf862e820, "ldr x0, [x1, x2, sxtx]"), + (0x4c, 0xf862f820, "ldr x0, [x1, x2, sxtx #3]"), + (0x50, 0xb8627820, "ldr w0, [x1, x2, lsl #2]"), + (0x54, 0xb87fd820, "ldr w0, [x1, wzr, sxtw #2]"), + (0x58, 0x397ffc20, "ldrb w0, [x1, #4095]"), + (0x5c, 0x385ff020, "ldurb w0, [x1, #-1]"), + (0x60, 0x38626820, "ldrb w0, [x1, x2]"), + (0x64, 0x3862c820, "ldrb w0, [x1, w2, sxtw]"), + (0x68, 0x38401420, "ldrb w0, [x1], #1"), + (0x6c, 0x38401c3f, "ldrb wzr, [x1, #1]!"), + (0x70, 0x797ffc20, "ldrh w0, [x1, #8190]"), + (0x74, 0x78403020, "ldurh w0, [x1, #3]"), + (0x78, 0x78627820, "ldrh w0, [x1, x2, lsl #1]"), + (0x7c, 0x39c01420, "ldrsb w0, [x1, #5]"), + (0x80, 0x39801420, "ldrsb x0, [x1, #5]"), + (0x84, 0x389fb020, "ldursb x0, [x1, #-5]"), + (0x88, 0x38e26820, "ldrsb w0, [x1, x2]"), + (0x8c, 0x79c00c20, "ldrsh w0, [x1, #6]"), + (0x90, 0x79800c20, "ldrsh x0, [x1, #6]"), + (0x94, 0x78807020, "ldursh x0, [x1, #7]"), + (0x98, 0x78a27820, "ldrsh x0, [x1, x2, lsl #1]"), + (0x9c, 0xb9800420, "ldrsw x0, [x1, #4]"), + (0xa0, 0xb89fc020, "ldursw x0, [x1, #-4]"), + (0xa4, 0xb8804c20, "ldrsw x0, [x1, #4]!"), + (0xa8, 0xb8804420, "ldrsw x0, [x1], #4"), + (0xac, 0xb8a27820, "ldrsw x0, [x1, x2, lsl #2]"), + (0xb0, 0xf9000020, "str x0, [x1]"), + (0xb4, 0xf9000bff, "str xzr, [sp, #16]"), + (0xb8, 0xb81fc03f, "stur wzr, [x1, #-4]"), + (0xbc, 0xb81fcc20, "str w0, [x1, #-4]!"), + (0xc0, 0xf8008420, "str x0, [x1], #8"), + (0xc4, 0xf8227820, "str x0, [x1, x2, lsl #3]"), + (0xc8, 0x39000420, "strb w0, [x1, #1]"), + (0xcc, 0x381ff020, "sturb w0, [x1, #-1]"), + (0xd0, 0x38226820, "strb w0, [x1, x2]"), + (0xd4, 0x79000420, "strh w0, [x1, #2]"), + (0xd8, 0x78001020, "sturh w0, [x1, #1]"), + (0xdc, 0x781fe43f, "strh wzr, [x1], #-2"), + (0xe0, 0xa9400440, "ldp x0, x1, [x2]"), + (0xe4, 0xa9408440, "ldp x0, x1, [x2, #8]"), + (0xe8, 0xa8c17bfd, "ldp x29, x30, [sp], #16"), + (0xec, 0xa9bf7bfd, "stp x29, x30, [sp, #-16]!"), + (0xf0, 0xa91ffbfd, "stp x29, x30, [sp, #504]"), + (0xf4, 0xa9207bfd, "stp x29, x30, [sp, #-512]"), + (0xf8, 0x295f8440, "ldp w0, w1, [x2, #252]"), + (0xfc, 0x29600440, "ldp w0, w1, [x2, #-256]"), + (0x100, 0x28817c40, "stp w0, wzr, [x2], #8"), + (0x104, 0x69400440, "ldpsw x0, x1, [x2]"), + (0x108, 0x697f8440, "ldpsw x0, x1, [x2, #-4]"), + (0x10c, 0x68df8440, "ldpsw x0, x1, [x2], #252"), + (0x110, 0x18000040, "ldr w0, 0x118"), + (0x114, 0x58ffffc0, "ldr x0, 0x10c"), + (0x118, 0x587fffff, "ldr xzr, 0x100114"), + (0x11c, 0x58800000, "ldr x0, 0xfffffffffff0011c"), + (0x120, 0x1c000040, "ldr s0, 0x128"), + (0x124, 0x5c000040, "ldr d0, 0x12c"), + (0x128, 0x9cffffdf, "ldr q31, 0x120"), + (0x12c, 0xd8000040, "prfm pldl1keep, 0x134"), + (0x130, 0xd8ffffd5, "prfm pstl3strm, 0x128"), + (0x134, 0xd8000059, "prfm #0x19, 0x13c"), + (0x138, 0xd8000058, "prfm #0x18, 0x140"), + (0x13c, 0xf9800000, "prfm pldl1keep, [x0]"), + (0x140, 0xf98007e6, "prfm pldslckeep, [sp, #8]"), + (0x144, 0xf9bffc0b, "prfm plil2strm, [x0, #32760]"), + (0x148, 0xf8a16810, "prfm pstl1keep, [x0, x1]"), + (0x14c, 0xf8a17813, "prfm pstl2strm, [x0, x1, lsl #3]"), + (0x150, 0xf8a14804, "prfm pldl3keep, [x0, w1, uxtw]"), + (0x154, 0xf8a1d804, "prfm pldl3keep, [x0, w1, sxtw #3]"), + (0x158, 0xf980041f, "prfm #0x1f, [x0, #8]"), + (0x15c, 0xf980041f, "prfm #0x1f, [x0, #8]"), + (0x160, 0xf9800418, "prfm ir, [x0, #8]"), + (0x164, 0xf89f8000, "prfum pldl1keep, [x0, #-8]"), + (0x168, 0xf8801011, "prfum pstl1strm, [x0, #1]"), + (0x16c, 0xf89ff018, "prfum #0x18, [x0, #-1]"), + (0x170, 0xf88ff3ff, "prfum #0x1f, [sp, #255]"), + (0x174, 0x885f7c20, "ldxr w0, [x1]"), + (0x178, 0xc85f7fe0, "ldxr x0, [sp]"), + (0x17c, 0xc8027c20, "stxr w2, x0, [x1]"), + (0x180, 0x88027fe0, "stxr w2, w0, [sp]"), + (0x184, 0x085f7c20, "ldxrb w0, [x1]"), + (0x188, 0x08027c20, "stxrb w2, w0, [x1]"), + (0x18c, 0x485f7c20, "ldxrh w0, [x1]"), + (0x190, 0x48027c20, "stxrh w2, w0, [x1]"), + (0x194, 0xb8200041, "ldadd w0, w1, [x2]"), + (0x198, 0xf82003e1, "ldadd x0, x1, [sp]"), + (0x19c, 0x38200041, "ldaddb w0, w1, [x2]"), + (0x1a0, 0x78200041, "ldaddh w0, w1, [x2]"), + (0x1a4, 0xf8201041, "ldclr x0, x1, [x2]"), + (0x1a8, 0xb8202041, "ldeor w0, w1, [x2]"), + (0x1ac, 0xb8203041, "ldset w0, w1, [x2]"), + (0x1b0, 0xb8204041, "ldsmax w0, w1, [x2]"), + (0x1b4, 0xf8205041, "ldsmin x0, x1, [x2]"), + (0x1b8, 0xb8206041, "ldumax w0, w1, [x2]"), + (0x1bc, 0xf8207041, "ldumin x0, x1, [x2]"), + (0x1c0, 0x38204041, "ldsmaxb w0, w1, [x2]"), + (0x1c4, 0x78207041, "lduminh w0, w1, [x2]"), + (0x1c8, 0xb820005f, "stadd w0, [x2]"), + (0x1cc, 0xf820005f, "stadd x0, [x2]"), + (0x1d0, 0x3820105f, "stclrb w0, [x2]"), + (0x1d4, 0xb8208041, "swp w0, w1, [x2]"), + (0x1d8, 0xf820805f, "swp x0, xzr, [x2]"), + (0x1dc, 0x38208041, "swpb w0, w1, [x2]"), + (0x1e0, 0x783f8041, "swph wzr, w1, [x2]"), + (0x1e4, 0x88a07c41, "cas w0, w1, [x2]"), + (0x1e8, 0xc8a07fe1, "cas x0, x1, [sp]"), + (0x1ec, 0x08a07c41, "casb w0, w1, [x2]"), + (0x1f0, 0x48a07c5f, "cash w0, wzr, [x2]"), + (0x1f4, 0x08207c82, "casp w0, w1, w2, w3, [x4]"), + (0x1f8, 0x48207c82, "casp x0, x1, x2, x3, [x4]"), + (0x1fc, 0x483e7c82, "casp x30, xzr, x2, x3, [x4]"), + (0x200, 0x48227c9e, "casp x2, x3, x30, xzr, [x4]"), + (0x204, 0x887f0440, "ldxp w0, w1, [x2]"), + (0x208, 0xc87f07e0, "ldxp x0, x1, [sp]"), + (0x20c, 0x88200861, "stxp w0, w1, w2, [x3]"), + (0x210, 0xc8200861, "stxp w0, x1, x2, [x3]"), + (0x214, 0xc8207fff, "stxp w0, xzr, xzr, [sp]"), + (0x218, 0x2d400440, "ldp s0, s1, [x2]"), + (0x21c, 0x6d408440, "ldp d0, d1, [x2, #8]"), + (0x220, 0xadc08440, "ldp q0, q1, [x2, #16]!"), + (0x224, 0xacbf7c40, "stp q0, q31, [x2], #-32"), + (0x228, 0x2d2007e0, "stp s0, s1, [sp, #-256]"), + (0x22c, 0x6d1f805f, "stp d31, d0, [x2, #504]"), + (0x230, 0xad1f8440, "stp q0, q1, [x2, #1008]"), + (0x234, 0x3d400020, "ldr b0, [x1]"), + (0x238, 0x7d400420, "ldr h0, [x1, #2]"), + (0x23c, 0xbd400420, "ldr s0, [x1, #4]"), + (0x240, 0xfd400420, "ldr d0, [x1, #8]"), + (0x244, 0x3dc00420, "ldr q0, [x1, #16]"), + (0x248, 0x3dfffc20, "ldr q0, [x1, #65520]"), + (0x24c, 0x3c5ff020, "ldur b0, [x1, #-1]"), + (0x250, 0x7c401020, "ldur h0, [x1, #1]"), + (0x254, 0xbc402020, "ldur s0, [x1, #2]"), + (0x258, 0xfc5f8020, "ldur d0, [x1, #-8]"), + (0x25c, 0x3cdf0020, "ldur q0, [x1, #-16]"), + (0x260, 0x3cc08020, "ldur q0, [x1, #8]"), + (0x264, 0x3c9f0fe0, "str q0, [sp, #-16]!"), + (0x268, 0xfc008420, "str d0, [x1], #8"), + (0x26c, 0x3c001c20, "str b0, [x1, #1]!"), + (0x270, 0x3ca27820, "str q0, [x1, x2, lsl #4]"), + (0x274, 0x3ca26820, "str q0, [x1, x2]"), + (0x278, 0xfc22d820, "str d0, [x1, w2, sxtw #3]"), + (0x27c, 0xbc224820, "str s0, [x1, w2, uxtw]"), + (0x280, 0x7c227820, "str h0, [x1, x2, lsl #1]"), + (0x284, 0x3c226820, "str b0, [x1, x2]"), + (0x288, 0x3c9f0020, "stur q0, [x1, #-16]"), + (0x290, 0x4c40a000, "ld1 {v0.16b-v1.16b}, [x0]"), + (0x294, 0x4cdfa000, "ld1 {v0.16b-v1.16b}, [x0], #32"), + (0x298, 0x4cc1a000, "ld1 {v0.16b-v1.16b}, [x0], x1"), + (0x29c, 0x4cc17400, "ld1 {v0.8h}, [x0], x1"), + (0x2a0, 0x0cc17c00, "ld1 {v0.1d}, [x0], x1"), + (0x2a4, 0x0cdf63e0, "ld1 {v0.8b-v2.8b}, [sp], #24"), + (0x2a8, 0x4cdf2800, "ld1 {v0.4s-v3.4s}, [x0], #64"), + (0x2ac, 0x4c40281f, "ld1 {v31.4s-v2.4s}, [x0]"), + (0x2b0, 0x4cc32c1e, "ld1 {v30.2d-v1.2d}, [x0], x3"), + (0x2b4, 0x0c40a800, "ld1 {v0.2s-v1.2s}, [x0]"), + (0x2b8, 0x0cdf2400, "ld1 {v0.4h-v3.4h}, [x0], #32"), + (0x2bc, 0x4c408800, "ld2 {v0.4s-v1.4s}, [x0]"), + (0x2c0, 0x4cdf8000, "ld2 {v0.16b-v1.16b}, [x0], #32"), + (0x2c4, 0x0cc28000, "ld2 {v0.8b-v1.8b}, [x0], x2"), + (0x2c8, 0x4cdf4c00, "ld3 {v0.2d-v2.2d}, [x0], #48"), + (0x2cc, 0x4c40441f, "ld3 {v31.8h-v1.8h}, [x0]"), + (0x2d0, 0x4c400000, "ld4 {v0.16b-v3.16b}, [x0]"), + (0x2d4, 0x0cc5001e, "ld4 {v30.8b-v1.8b}, [x0], x5"), + (0x2d8, 0x0cdf07e0, "ld4 {v0.4h-v3.4h}, [sp], #32"), + (0x2dc, 0x4c00a000, "st1 {v0.16b-v1.16b}, [x0]"), + (0x2e0, 0x4c9f2800, "st1 {v0.4s-v3.4s}, [x0], #64"), + (0x2e4, 0x0c812000, "st1 {v0.8b-v3.8b}, [x0], x1"), + (0x2e8, 0x0c9f83e0, "st2 {v0.8b-v1.8b}, [sp], #16"), + (0x2ec, 0x4c008c00, "st2 {v0.2d-v1.2d}, [x0]"), + (0x2f0, 0x4c864800, "st3 {v0.4s-v2.4s}, [x0], x6"), + (0x2f4, 0x4c9f001d, "st4 {v29.16b-v0.16b}, [x0], #64"), + (0x2f8, 0x0d400000, "ld1 {v0.b}[0], [x0]"), + (0x2fc, 0x4d401c00, "ld1 {v0.b}[15], [x0]"), + (0x300, 0x0ddf1c00, "ld1 {v0.b}[7], [x0], #1"), + (0x304, 0x4d405800, "ld1 {v0.h}[7], [x0]"), + (0x308, 0x0ddf4000, "ld1 {v0.h}[0], [x0], #2"), + (0x30c, 0x4dc29000, "ld1 {v0.s}[3], [x0], x2"), + (0x310, 0x0ddf8000, "ld1 {v0.s}[0], [x0], #4"), + (0x314, 0x4dc28400, "ld1 {v0.d}[1], [x0], x2"), + (0x318, 0x0ddf841f, "ld1 {v31.d}[0], [x0], #8"), + (0x31c, 0x0d600c00, "ld2 {v0.b-v1.b}[3], [x0]"), + (0x320, 0x0dff5800, "ld2 {v0.h-v1.h}[3], [x0], #4"), + (0x324, 0x0de49000, "ld2 {v0.s-v1.s}[1], [x0], x4"), + (0x328, 0x4dff8400, "ld2 {v0.d-v1.d}[1], [x0], #16"), + (0x32c, 0x0ddfb000, "ld3 {v0.s-v2.s}[1], [x0], #12"), + (0x330, 0x4d40241f, "ld3 {v31.b-v1.b}[9], [x0]"), + (0x334, 0x0d60a400, "ld4 {v0.d-v3.d}[0], [x0]"), + (0x338, 0x0dff73e0, "ld4 {v0.h-v3.h}[2], [sp], #8"), + (0x33c, 0x4d008000, "st1 {v0.s}[2], [x0]"), + (0x340, 0x0d9f0400, "st1 {v0.b}[1], [x0], #1"), + (0x344, 0x4da38400, "st2 {v0.d-v1.d}[1], [x0], x3"), + (0x348, 0x4d9f6800, "st3 {v0.h-v2.h}[5], [x0], #6"), + (0x34c, 0x4d20b000, "st4 {v0.s-v3.s}[3], [x0]"), + (0x350, 0x4d40c000, "ld1r {v0.16b}, [x0]"), + (0x354, 0x0ddfc000, "ld1r {v0.8b}, [x0], #1"), + (0x358, 0x4ddfc800, "ld1r {v0.4s}, [x0], #4"), + (0x35c, 0x4dc1cc00, "ld1r {v0.2d}, [x0], x1"), + (0x360, 0x0ddfc400, "ld1r {v0.4h}, [x0], #2"), + (0x364, 0x4d60c800, "ld2r {v0.4s-v1.4s}, [x0]"), + (0x368, 0x4dffc400, "ld2r {v0.8h-v1.8h}, [x0], #4"), + (0x36c, 0x0de3cc00, "ld2r {v0.1d-v1.1d}, [x0], x3"), + (0x370, 0x0ddfe800, "ld3r {v0.2s-v2.2s}, [x0], #12"), + (0x374, 0x4d40ec1f, "ld3r {v31.2d-v1.2d}, [x0]"), + (0x378, 0x4d60ec00, "ld4r {v0.2d-v3.2d}, [x0]"), + (0x37c, 0x4dffe3e0, "ld4r {v0.16b-v3.16b}, [sp], #4"), + (0x380, 0x4de7e400, "ld4r {v0.8h-v3.8h}, [x0], x7"), + (0x384, 0x98000040, "ldrsw x0, 0x38c"), + (0x388, 0x98ffffc3, "ldrsw x3, 0x380"), + (0x38c, 0x9800005f, "ldrsw xzr, 0x394"), + ]; + + #[test] + fn matches_objdump() { + for &(pc, word, expected) in ROWS { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:#010x}: {e:?}")); + assert_eq!( + instruction.display_at(pc).to_string(), + expected, + "{word:#010x}" + ); + } + } + + /// `(word, text)`: GNU objdump 2.46's spelling of the words whose identity the decoded + /// form must keep (acquire/release, unscaled and unprivileged offsets, non-temporal + /// pairs, `ld1` arrangements, byte register offsets, `rprfm`). + const IDENTITY_ROWS: &[(u32, &str)] = &[ + (0x88dffc20, "ldar w0, [x1]"), + (0xc8dfffe0, "ldar x0, [sp]"), + (0x08dffc20, "ldarb w0, [x1]"), + (0x48dffc20, "ldarh w0, [x1]"), + (0x88df7c20, "ldlar w0, [x1]"), + (0xc8df7c20, "ldlar x0, [x1]"), + (0x08df7c20, "ldlarb w0, [x1]"), + (0x48df7c20, "ldlarh w0, [x1]"), + (0x889f7c20, "stllr w0, [x1]"), + (0xc89f7c20, "stllr x0, [x1]"), + (0x089f7c20, "stllrb w0, [x1]"), + (0x489f7c20, "stllrh w0, [x1]"), + (0x889ffc20, "stlr w0, [x1]"), + (0xc89fffe0, "stlr x0, [sp]"), + (0x089ffc20, "stlrb w0, [x1]"), + (0x489ffc20, "stlrh w0, [x1]"), + (0x889ffc1f, "stlr wzr, [x0]"), + (0xb8bfc020, "ldapr w0, [x1]"), + (0xf8bfc020, "ldapr x0, [x1]"), + (0x38bfc020, "ldaprb w0, [x1]"), + (0x78bfc020, "ldaprh w0, [x1]"), + (0x99404020, "ldapur w0, [x1, #4]"), + (0xd95f8020, "ldapur x0, [x1, #-8]"), + (0x19404020, "ldapurb w0, [x1, #4]"), + (0x59400020, "ldapurh w0, [x1]"), + (0x19c04020, "ldapursb w0, [x1, #4]"), + (0x19804020, "ldapursb x0, [x1, #4]"), + (0x59c04020, "ldapursh w0, [x1, #4]"), + (0x59804020, "ldapursh x0, [x1, #4]"), + (0x99804020, "ldapursw x0, [x1, #4]"), + (0x99004020, "stlur w0, [x1, #4]"), + (0xd9100020, "stlur x0, [x1, #-256]"), + (0x19000020, "stlurb w0, [x1]"), + (0x590ff020, "stlurh w0, [x1, #255]"), + (0x885ffc20, "ldaxr w0, [x1]"), + (0xc85fffe0, "ldaxr x0, [sp]"), + (0x085ffc20, "ldaxrb w0, [x1]"), + (0x485ffc20, "ldaxrh w0, [x1]"), + (0x8802fc20, "stlxr w2, w0, [x1]"), + (0xc802ffe0, "stlxr w2, x0, [sp]"), + (0x0802fc20, "stlxrb w2, w0, [x1]"), + (0x4802fc20, "stlxrh w2, w0, [x1]"), + (0x887f8440, "ldaxp w0, w1, [x2]"), + (0xc87f87e0, "ldaxp x0, x1, [sp]"), + (0x88238440, "stlxp w3, w0, w1, [x2]"), + (0xc8238440, "stlxp w3, x0, x1, [x2]"), + (0x88e07c41, "casa w0, w1, [x2]"), + (0x88a0fc41, "casl w0, w1, [x2]"), + (0xc8e0fc41, "casal x0, x1, [x2]"), + (0x08e07c41, "casab w0, w1, [x2]"), + (0x48e0fc41, "casalh w0, w1, [x2]"), + (0x08a0fc5f, "caslb w0, wzr, [x2]"), + (0x08607c82, "caspa w0, w1, w2, w3, [x4]"), + (0x4820fc82, "caspl x0, x1, x2, x3, [x4]"), + (0x4860fc82, "caspal x0, x1, x2, x3, [x4]"), + (0xb8a08041, "swpa w0, w1, [x2]"), + (0xf8608041, "swpl x0, x1, [x2]"), + (0xf8e0805f, "swpal x0, xzr, [x2]"), + (0x38a08041, "swpab w0, w1, [x2]"), + (0x78608041, "swplh w0, w1, [x2]"), + (0x38e08041, "swpalb w0, w1, [x2]"), + (0xb8a00041, "ldadda w0, w1, [x2]"), + (0xb8600041, "ldaddl w0, w1, [x2]"), + (0xf8e00041, "ldaddal x0, x1, [x2]"), + (0xb8a0003f, "ldadda w0, wzr, [x1]"), + (0xf8e0003f, "ldaddal x0, xzr, [x1]"), + (0xb860003f, "staddl w0, [x1]"), + (0x38a0003f, "ldaddab w0, wzr, [x1]"), + (0x7860003f, "staddlh w0, [x1]"), + (0x38e0003f, "ldaddalb w0, wzr, [x1]"), + (0x3860003f, "staddlb w0, [x1]"), + (0xb8a01041, "ldclra w0, w1, [x2]"), + (0xb8602041, "ldeorl w0, w1, [x2]"), + (0xb8a0305f, "ldseta w0, wzr, [x2]"), + (0xf8e04041, "ldsmaxal x0, x1, [x2]"), + (0xb8605041, "ldsminl w0, w1, [x2]"), + (0x78e06041, "ldumaxalh w0, w1, [x2]"), + (0x38a07041, "lduminab w0, w1, [x2]"), + (0xb8400020, "ldur w0, [x1]"), + (0xb8408020, "ldur w0, [x1, #8]"), + (0xf8410020, "ldur x0, [x1, #16]"), + (0xf85f8020, "ldur x0, [x1, #-8]"), + (0x38408020, "ldurb w0, [x1, #8]"), + (0x78404020, "ldurh w0, [x1, #4]"), + (0x38808020, "ldursb x0, [x1, #8]"), + (0x78c08020, "ldursh w0, [x1, #8]"), + (0xb8808020, "ldursw x0, [x1, #8]"), + (0xf8008020, "stur x0, [x1, #8]"), + (0x38001020, "sturb w0, [x1, #1]"), + (0xf9400420, "ldr x0, [x1, #8]"), + (0xf9400020, "ldr x0, [x1]"), + (0xb8404820, "ldtr w0, [x1, #4]"), + (0xf8400820, "ldtr x0, [x1]"), + (0x385ff820, "ldtrb w0, [x1, #-1]"), + (0x78402820, "ldtrh w0, [x1, #2]"), + (0x38c00820, "ldtrsb w0, [x1]"), + (0x78900820, "ldtrsh x0, [x1, #-256]"), + (0xb8804820, "ldtrsw x0, [x1, #4]"), + (0xf8008820, "sttr x0, [x1, #8]"), + (0x38000820, "sttrb w0, [x1]"), + (0x780ff820, "sttrh w0, [x1, #255]"), + (0xf9800020, "prfm pldl1keep, [x1]"), + (0xf8800020, "prfum pldl1keep, [x1]"), + (0xf8808020, "prfum pldl1keep, [x1, #8]"), + (0xf9800420, "prfm pldl1keep, [x1, #8]"), + (0xf89f8033, "prfum pstl2strm, [x1, #-8]"), + (0xf8a26820, "prfm pldl1keep, [x1, x2]"), + (0xf8a27820, "prfm pldl1keep, [x1, x2, lsl #3]"), + (0xf8a24820, "prfm pldl1keep, [x1, w2, uxtw]"), + (0xf8a2d820, "prfm pldl1keep, [x1, w2, sxtw #3]"), + (0xf8a2e820, "prfm pldl1keep, [x1, x2, sxtx]"), + (0xf8a24838, "rprfm pldkeep, x2, [x1]"), + (0xf8a24839, "rprfm pstkeep, x2, [x1]"), + (0xf8bf4bfc, "rprfm pldstrm, xzr, [sp]"), + (0xf8a2483d, "rprfm pststrm, x2, [x1]"), + (0xf8bf483c, "rprfm pldstrm, xzr, [x1]"), + (0xf8bf4bfd, "rprfm pststrm, xzr, [sp]"), + (0xf8a2483a, "rprfm #2, x2, [x1]"), + (0x38627820, "ldrb w0, [x1, x2, lsl #0]"), + (0x38626820, "ldrb w0, [x1, x2]"), + (0x38625820, "ldrb w0, [x1, w2, uxtw #0]"), + (0x3862d820, "ldrb w0, [x1, w2, sxtw #0]"), + (0x3862c820, "ldrb w0, [x1, w2, sxtw]"), + (0x3862f820, "ldrb w0, [x1, x2, sxtx #0]"), + (0x38a27820, "ldrsb x0, [x1, x2, lsl #0]"), + (0x38227820, "strb w0, [x1, x2, lsl #0]"), + (0x3c207962, "str b2, [x11, x0, lsl #0]"), + (0x3c206962, "str b2, [x11, x0]"), + (0x3c605962, "ldr b2, [x11, w0, uxtw #0]"), + (0x3ca07962, "str q2, [x11, x0, lsl #4]"), + (0x3ca06962, "str q2, [x11, x0]"), + (0x7c607962, "ldr h2, [x11, x0, lsl #1]"), + (0xf8607962, "ldr x2, [x11, x0, lsl #3]"), + (0xa8400440, "ldnp x0, x1, [x2]"), + (0x28600440, "ldnp w0, w1, [x2, #-256]"), + (0xa8008440, "stnp x0, x1, [x2, #8]"), + (0x28010440, "stnp w0, w1, [x2, #8]"), + (0x2c400440, "ldnp s0, s1, [x2]"), + (0x6c408440, "ldnp d0, d1, [x2, #8]"), + (0xac408440, "ldnp q0, q1, [x2, #16]"), + (0x2c000440, "stnp s0, s1, [x2]"), + (0x6c3f8440, "stnp d0, d1, [x2, #-8]"), + (0xac3f0440, "stnp q0, q1, [x2, #-32]"), + (0x4c407020, "ld1 {v0.16b}, [x1]"), + (0x0c407020, "ld1 {v0.8b}, [x1]"), + (0x4c407420, "ld1 {v0.8h}, [x1]"), + (0x0c407420, "ld1 {v0.4h}, [x1]"), + (0x4c407820, "ld1 {v0.4s}, [x1]"), + (0x0c407820, "ld1 {v0.2s}, [x1]"), + (0x4c407c20, "ld1 {v0.2d}, [x1]"), + (0x0c407c20, "ld1 {v0.1d}, [x1]"), + (0x4c4073ff, "ld1 {v31.16b}, [sp]"), + (0x4cdf7020, "ld1 {v0.16b}, [x1], #16"), + (0x0cdf7020, "ld1 {v0.8b}, [x1], #8"), + (0x0cdf7c20, "ld1 {v0.1d}, [x1], #8"), + (0x4cdf7820, "ld1 {v0.4s}, [x1], #16"), + (0x0cc27420, "ld1 {v0.4h}, [x1], x2"), + (0x4cc27020, "ld1 {v0.16b}, [x1], x2"), + (0x4c007020, "st1 {v0.16b}, [x1]"), + (0x4c007420, "st1 {v0.8h}, [x1]"), + (0x0c9f7820, "st1 {v0.2s}, [x1], #8"), + (0x4c9f73ff, "st1 {v31.16b}, [sp], #16"), + (0x0c007c20, "st1 {v0.1d}, [x1]"), + (0x3dc00020, "ldr q0, [x1]"), + (0xfd400020, "ldr d0, [x1]"), + (0x3cc10020, "ldur q0, [x1, #16]"), + (0x3cc08020, "ldur q0, [x1, #8]"), + (0xfc5f8020, "ldur d0, [x1, #-8]"), + (0xfc400020, "ldur d0, [x1]"), + (0xbc404020, "ldur s0, [x1, #4]"), + (0x7c402020, "ldur h0, [x1, #2]"), + (0x3c400020, "ldur b0, [x1]"), + (0x3c401020, "ldur b0, [x1, #1]"), + (0x3d400420, "ldr b0, [x1, #1]"), + (0x3c810020, "stur q0, [x1, #16]"), + (0xfc1f8020, "stur d0, [x1, #-8]"), + (0xfd400420, "ldr d0, [x1, #8]"), + ]; + + #[test] + fn keeps_instruction_identity() { + for &(word, expected) in IDENTITY_ROWS { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:#010x}: {e:?}")); + assert_eq!(instruction.display().to_string(), expected, "{word:#010x}"); + } + } + + #[test] + fn without_pc_targets_are_relative() { + assert_eq!( + decode(0x18000040).unwrap().display().to_string(), + "ldr w0, .+8" + ); + assert_eq!( + decode(0xd8ffffd5).unwrap().display().to_string(), + "prfm pstl3strm, .-8" + ); + } +} diff --git a/src/isa/aarch64/src/format/mod.rs b/src/isa/aarch64/src/format/mod.rs new file mode 100644 index 0000000..71e6bc9 --- /dev/null +++ b/src/isa/aarch64/src/format/mod.rs @@ -0,0 +1,211 @@ +//! Textual disassembly of decoded instructions, in GNU binutils (`objdump`) spelling. +//! +//! The text is a presentation of an already decoded [`Instruction`]: nothing here +//! decodes bits. Assembly aliases (`mov`, `cmp`, `ret`, `lsl #n`, ...) are chosen +//! here, from the decoded fields, the way the Arm ARM's alias conditions describe. +//! +//! * [`Instruction::display`] renders without an address: PC-relative operands +//! print as `.+N` / `.-N`. +//! * [`Instruction::display_at`] renders at `pc`: targets print as absolute +//! addresses (`0x1008`). +//! * [`Instruction::display_with`] additionally annotates targets with the symbol +//! that contains them (`0x1008 `), through [`Symbols`]. +//! * [`word`], [`word_at`] and [`word_with`] start from a raw word and render +//! `.word 0x...` for one the decoder does not recognise. + +use crate::decode::{Instruction, decode}; +use core::fmt::{self, Display, Formatter}; + +mod fp; +mod integer; +mod memory; +mod reg; +mod system; +mod vector; + +/// Resolves a code or data address to the symbol that contains it. +pub trait Symbols { + /// The symbol containing `addr` and the byte offset of `addr` into it. + fn lookup(&self, addr: u64) -> Option<(&str, u64)>; +} + +/// No symbols: targets print bare. +impl Symbols for () { + fn lookup(&self, _: u64) -> Option<(&str, u64)> { + None + } +} + +/// Where an instruction is, and what is known about its surroundings. +#[derive(Clone, Copy)] +pub(crate) struct Context<'a> { + /// The instruction's address, when known. Without it a PC-relative operand + /// prints as an offset from `.`. + pub pc: Option, + pub symbols: &'a dyn Symbols, +} + +impl Context<'_> { + /// A PC-relative code or data target `offset` bytes from the instruction: + /// `.+N` / `.-N` without a pc, else `0x` followed by ` ` or + /// ` ` when a symbol contains it. Address arithmetic wraps; a + /// disassembler must never panic on the bytes it is given. + pub fn target(&self, f: &mut Formatter<'_>, offset: i64) -> fmt::Result { + match self.pc { + None => { + let sign = if offset < 0 { '-' } else { '+' }; + write!(f, ".{sign}{}", offset.unsigned_abs()) + } + Some(pc) => self.address(f, pc.wrapping_add(offset as u64)), + } + } + + /// Like [`Self::target`] for `adrp`: `offset` is relative to the 4 KiB page + /// containing the instruction. + pub fn page_target(&self, f: &mut Formatter<'_>, offset: i64) -> fmt::Result { + match self.pc { + None => { + let sign = if offset < 0 { '-' } else { '+' }; + write!(f, ".{sign}{}", offset.unsigned_abs()) + } + Some(pc) => self.address(f, (pc & !0xfff).wrapping_add(offset as u64)), + } + } + + /// An absolute address with its symbol annotation. + fn address(&self, f: &mut Formatter<'_>, addr: u64) -> fmt::Result { + write!(f, "{addr:#x}")?; + match self.symbols.lookup(addr) { + Some((name, 0)) => write!(f, " <{name}>"), + Some((name, offset)) => write!(f, " <{name}+{offset:#x}>"), + None => Ok(()), + } + } +} + +/// An [`Instruction`] ready to print; see [`Instruction::display`]. +pub struct Disassembly<'a> { + instruction: &'a Instruction, + cx: Context<'a>, +} + +impl Instruction { + /// Render without an address. + pub fn display(&self) -> Disassembly<'_> { + Disassembly { + instruction: self, + cx: Context { + pc: None, + symbols: &(), + }, + } + } + + /// Render at address `pc`. + pub fn display_at(&self, pc: u64) -> Disassembly<'_> { + Disassembly { + instruction: self, + cx: Context { + pc: Some(pc), + symbols: &(), + }, + } + } + + /// Render at address `pc`, annotating targets from `symbols`. + pub fn display_with<'a>(&'a self, pc: u64, symbols: &'a dyn Symbols) -> Disassembly<'a> { + Disassembly { + instruction: self, + cx: Context { + pc: Some(pc), + symbols, + }, + } + } +} + +impl Display for Disassembly<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write(f, &self.cx, self.instruction) + } +} + +/// A raw instruction word ready to print; see [`word`]. +pub struct Word<'a> { + word: u32, + cx: Context<'a>, +} + +/// Render `word` without an address; `.word 0x...` when it does not decode. +pub fn word(word: u32) -> Word<'static> { + Word { + word, + cx: Context { + pc: None, + symbols: &(), + }, + } +} + +/// Render `word` at `pc`; `.word 0x...` when it does not decode. +pub fn word_at(word: u32, pc: u64) -> Word<'static> { + Word { + word, + cx: Context { + pc: Some(pc), + symbols: &(), + }, + } +} + +/// Render `word` at `pc`, annotating targets from `symbols`. +pub fn word_with(word: u32, pc: u64, symbols: &dyn Symbols) -> Word<'_> { + Word { + word, + cx: Context { + pc: Some(pc), + symbols, + }, + } +} + +impl Display for Word<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match decode(self.word) { + Ok(instruction) => write(f, &self.cx, &instruction), + Err(_) => write!(f, ".word {:#010x}", self.word), + } + } +} + +/// Route `instruction` to the module that owns its family. Every variant is named +/// here, so a new one does not compile until it has a formatter. +fn write(f: &mut Formatter<'_>, cx: &Context<'_>, instruction: &Instruction) -> fmt::Result { + use Instruction::*; + match instruction { + Movz(_) | Movn(_) | Movk(_) | ArithImm(_) | ArithReg(_) | ArithExt(_) | LogicReg(_) + | LogicImm(_) | Mul(_) | MulLong(_) | MulHigh(_) | Adr(_) | Csel(_) | Variable(_) + | CondCompare(_) | ArithCarry(_) | Unary(_) | Bitfield(_) | Extract(_) => { + integer::write(f, cx, instruction) + } + Memory(_) | Pair(_) | Literal(_) | SimdLiteral(_) | Exclusive(_) | Atomic(_) + | Prefetch(_) | PrefetchLiteral(_) | RangePrefetch(_) | SimdPair(_) | SimdMemory(_) + | SimdStruct(_) => memory::write(f, cx, instruction), + TestBranch(_) | Call(_) | Indirect(_) | B(_) | BCond(_) | Eret | System(_) | DcZva(_) + | AddressTranslate(_) | CacheOp(_) | Tlbi(_) | Clrex(_) | Nop | Hint(_) | Dsb(_) + | Dmb(_) | Isb(_) | Sev | Sevl | Wfe | Wfi | Yield | Brk(_) | Svc(_) | Psci | Pauth(_) + | Gxf(_) => system::write(f, cx, instruction), + SimdImm(_) + | SimdCompareZero(_) + | SimdDup { .. } + | SimdDupElement(_) + | SimdInsert(_) + | SimdInsertElement(_) + | SimdExtract(_) + | SimdExt(_) + | SimdAlu(_) + | SimdTable(_) + | Simd(_) => vector::write(f, cx, instruction), + Fp(_) | SimdFmov(_) => fp::write(f, cx, instruction), + } +} diff --git a/src/isa/aarch64/src/format/reg.rs b/src/isa/aarch64/src/format/reg.rs new file mode 100644 index 0000000..ece25a6 --- /dev/null +++ b/src/isa/aarch64/src/format/reg.rs @@ -0,0 +1,76 @@ +//! Register and condition spelling shared by every family. + +use crate::decode::{Condition, Width}; +use core::fmt::{self, Formatter}; + +/// A general register where encoding 31 is the zero register: `x0`..`x30`, `xzr`, +/// `w0`..`w30`, `wzr`. +pub fn gpr(f: &mut Formatter<'_>, width: Width, r: u8) -> fmt::Result { + let prefix = match width { + Width::W32 => 'w', + Width::X64 => 'x', + }; + if r == 31 { + write!(f, "{prefix}zr") + } else { + write!(f, "{prefix}{r}") + } +} + +/// A general register where encoding 31 is the stack pointer: `sp` / `wsp`. +pub fn gpr_sp(f: &mut Formatter<'_>, width: Width, r: u8) -> fmt::Result { + match (r, width) { + (31, Width::X64) => f.write_str("sp"), + (31, Width::W32) => f.write_str("wsp"), + _ => gpr(f, width, r), + } +} + +/// The condition code as binutils spells it (`cs`/`cc`, not `hs`/`lo`). +pub fn condition(c: Condition) -> &'static str { + match c { + Condition::Eq => "eq", + Condition::Ne => "ne", + Condition::Cs => "cs", + Condition::Cc => "cc", + Condition::Mi => "mi", + Condition::Pl => "pl", + Condition::Vs => "vs", + Condition::Vc => "vc", + Condition::Hi => "hi", + Condition::Ls => "ls", + Condition::Ge => "ge", + Condition::Lt => "lt", + Condition::Gt => "gt", + Condition::Le => "le", + Condition::Al => "al", + Condition::Nv => "nv", + } +} + +/// A SIMD&FP scalar register `bytes` wide: `b`, `h`, `s`, `d` or `q` for 1, 2, 4, 8, 16. +pub fn fp_scalar(f: &mut Formatter<'_>, bytes: u8, r: u8) -> fmt::Result { + let prefix = match bytes { + 1 => 'b', + 2 => 'h', + 4 => 's', + 8 => 'd', + _ => 'q', + }; + write!(f, "{prefix}{r}") +} + +/// The element suffix of a vector arrangement: `size` is log2 of the element width +/// in bytes (0 = bytes .. 3 = doublewords), `q` selects 128 bits over 64. +/// `(0, true)` is `16b`, `(2, false)` is `2s`, `(3, false)` is `1d`. +pub fn arrangement(f: &mut Formatter<'_>, size: u8, q: bool) -> fmt::Result { + let lanes = (if q { 16 } else { 8 }) >> size; + let element = ['b', 'h', 's', 'd'][usize::from(size & 3)]; + write!(f, "{lanes}{element}") +} + +/// A vector register with its arrangement: `v0.16b`. +pub fn vector(f: &mut Formatter<'_>, r: u8, size: u8, q: bool) -> fmt::Result { + write!(f, "v{r}.")?; + arrangement(f, size, q) +} diff --git a/src/isa/aarch64/src/format/system.rs b/src/isa/aarch64/src/format/system.rs new file mode 100644 index 0000000..3467381 --- /dev/null +++ b/src/isa/aarch64/src/format/system.rs @@ -0,0 +1,1259 @@ +//! Branches, system instructions and the families that are not data processing: +//! pointer authentication and guarded execution. +//! +//! Owns `TestBranch`, `Call`, `Indirect`, `B`, `BCond`, `Eret`, `System` (`mrs`/`msr`, +//! with every `SystemRegister` name), `DcZva`, `AddressTranslate`, `CacheOp`, `Tlbi`, +//! `Clrex`, `Nop`, `Hint`, `Dsb`, `Dmb`, `Isb`, `Sev`, `Sevl`, `Wfe`, `Wfi`, `Yield`, +//! `Brk`, `Svc`, `Psci`, `Pauth` and `Gxf`. + +use super::{ + Context, + reg::{condition, gpr}, +}; +use crate::decode::{self, IndirectKind, Instruction, SysOp, SystemRegister, Width}; +use crate::gxf; +use crate::pauth::{self, KeyId, Modifier}; +use core::fmt::{self, Formatter}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + use Instruction::*; + match *instruction { + TestBranch(test) => test_branch(f, cx, &test), + Call(call) => { + f.write_str(if call.link { "bl " } else { "b " })?; + cx.target(f, call.target) + } + B(offset) => { + f.write_str("b ")?; + cx.target(f, offset) + } + BCond(branch) => { + write!(f, "b.{} ", condition(branch.cond))?; + cx.target(f, branch.offset) + } + Indirect(indirect) => match indirect.kind { + IndirectKind::Jump => { + f.write_str("br ")?; + gpr(f, Width::X64, indirect.rn) + } + IndirectKind::Call => { + f.write_str("blr ")?; + gpr(f, Width::X64, indirect.rn) + } + IndirectKind::Return if indirect.rn == 30 => f.write_str("ret"), + IndirectKind::Return => { + f.write_str("ret ")?; + gpr(f, Width::X64, indirect.rn) + } + }, + Eret => f.write_str("eret"), + System(system) => system_register_move(f, &system), + DcZva(rt) => { + f.write_str("dc zva, ")?; + gpr(f, Width::X64, rt) + } + AddressTranslate(at) => { + write!( + f, + "at s1e{}{}, ", + if at.user { '0' } else { '1' }, + if at.write { 'w' } else { 'r' } + )?; + gpr(f, Width::X64, at.rt) + } + CacheOp(op) => cache_op(f, &op), + Tlbi(op) => tlbi(f, &op), + Clrex(option) => option_or_default(f, "clrex", option), + Nop => f.write_str("nop"), + Hint(n) => hint(f, n), + Dsb(option) => match option { + 0 => f.write_str("ssbb"), + 4 => f.write_str("pssbb"), + _ => barrier(f, "dsb", option), + }, + Dmb(option) => barrier(f, "dmb", option), + Isb(option) => option_or_default(f, "isb", option), + Sev => f.write_str("sev"), + Sevl => f.write_str("sevl"), + Wfe => f.write_str("wfe"), + Wfi => f.write_str("wfi"), + Yield => f.write_str("yield"), + Brk(imm) => write!(f, "brk #{imm:#x}"), + Svc(imm) => write!(f, "svc #{imm:#x}"), + Psci => f.write_str("hvc #0x0"), + Pauth(op) => pauth_op(f, op), + Gxf(op) => match op { + gxf::Op::Gexit => f.write_str(".inst 0x00201400"), + gxf::Op::Genter { imm5 } => write!(f, ".inst {:#010x}", 0x0020_1420 | u32::from(imm5)), + }, + _ => unreachable!("{instruction:?} is not a system instruction"), + } +} + +fn test_branch(f: &mut Formatter<'_>, cx: &Context<'_>, test: &decode::Test) -> fmt::Result { + match test.bit_index { + None => f.write_str(if test.negate { "cbnz " } else { "cbz " })?, + Some(_) => f.write_str(if test.negate { "tbnz " } else { "tbz " })?, + } + gpr(f, test.width, test.rt)?; + f.write_str(", ")?; + if let Some(bit) = test.bit_index { + write!(f, "#{bit}, ")?; + } + cx.target(f, test.offset) +} + +/// `mnemonic` alone for the architectural default option 15, else `mnemonic #imm`. +fn option_or_default(f: &mut Formatter<'_>, mnemonic: &str, option: u8) -> fmt::Result { + if option == 15 { + f.write_str(mnemonic) + } else { + write!(f, "{mnemonic} #{option:#x}") + } +} + +/// `dsb`/`dmb` with the barrier option named when binutils names it. +fn barrier(f: &mut Formatter<'_>, mnemonic: &str, option: u8) -> fmt::Result { + let name = match option { + 1 => "oshld", + 2 => "oshst", + 3 => "osh", + 5 => "nshld", + 6 => "nshst", + 7 => "nsh", + 9 => "ishld", + 10 => "ishst", + 11 => "ish", + 13 => "ld", + 14 => "st", + 15 => "sy", + _ => return write!(f, "{mnemonic} #{option:#04x}"), + }; + write!(f, "{mnemonic} {name}") +} + +fn hint(f: &mut Formatter<'_>, n: u8) -> fmt::Result { + let text = match n { + 0 => "nop", + 1 => "yield", + 2 => "wfe", + 3 => "wfi", + 4 => "sev", + 5 => "sevl", + 6 => "dgh", + // Hints 7, 8, 10, 12, 14 and 24 to 31 (`xpaclri`, `pacia1716`, `paciasp`, ...) are + // pointer authentication: the decoder yields `Pauth`, never `Hint`, for them. + 16 => "esb", + 17 => "psb csync", + 18 => "tsb csync", + 19 => "gcsb dsync", + 20 => "csdb", + 22 => "clrbhb", + 32 => "bti r", + 34 => "bti c", + 36 => "bti j", + 38 => "bti jc", + 40 => "chkfeat x16", + 48 => "stshh keep", + 49 => "stshh strm", + 50 => "shuh", + 51 => "shuh ph", + 52 => "stcph", + _ => return write!(f, "hint #{n:#x}"), + }; + f.write_str(text) +} + +/// How a named `sys` alias spells its `Xt` operand. +#[derive(Clone, Copy)] +enum Xt { + /// Never printed. + Never, + /// Always printed (`xzr` for 31). + Always, + /// Printed unless it is 31. + UnlessZero, +} + +const IC: &[(u8, u8, u8, &str, Xt)] = &[ + (0, 1, 0, "ialluis", Xt::Never), + (0, 5, 0, "iallu", Xt::Never), + (3, 5, 1, "ivau", Xt::Always), +]; + +const DC: &[(u8, u8, u8, &str, Xt)] = &[ + (0, 6, 1, "ivac", Xt::Always), + (0, 6, 2, "isw", Xt::Always), + (0, 6, 3, "igvac", Xt::Always), + (0, 6, 4, "igsw", Xt::Always), + (0, 6, 5, "igdvac", Xt::Always), + (0, 6, 6, "igdsw", Xt::Always), + (0, 10, 2, "csw", Xt::Always), + (0, 10, 4, "cgsw", Xt::Always), + (0, 10, 6, "cgdsw", Xt::Always), + (0, 14, 2, "cisw", Xt::Always), + (0, 14, 4, "cigsw", Xt::Always), + (0, 14, 6, "cigdsw", Xt::Always), + (3, 10, 1, "cvac", Xt::Always), + (3, 10, 3, "cgvac", Xt::Always), + (3, 10, 5, "cgdvac", Xt::Always), + (3, 11, 0, "cvaoc", Xt::Always), + (3, 11, 1, "cvau", Xt::Always), + (3, 11, 7, "cgdvaoc", Xt::Always), + (3, 14, 1, "civac", Xt::Always), + (3, 14, 3, "cigvac", Xt::Always), + (3, 14, 5, "cigdvac", Xt::Always), + (4, 14, 0, "cipae", Xt::Always), + (4, 14, 7, "cigdpae", Xt::Always), + (6, 14, 1, "cipapa", Xt::Always), + (6, 14, 5, "cigdpapa", Xt::Always), +]; + +const TLBI: &[(u8, u8, u8, &str, Xt)] = &[ + (0, 1, 0, "vmalle1os", Xt::UnlessZero), + (0, 1, 1, "vae1os", Xt::Always), + (0, 1, 2, "aside1os", Xt::Always), + (0, 1, 3, "vaae1os", Xt::Always), + (0, 1, 5, "vale1os", Xt::Always), + (0, 1, 7, "vaale1os", Xt::Always), + (0, 2, 1, "rvae1is", Xt::Always), + (0, 2, 3, "rvaae1is", Xt::Always), + (0, 2, 5, "rvale1is", Xt::Always), + (0, 2, 7, "rvaale1is", Xt::Always), + (0, 3, 0, "vmalle1is", Xt::UnlessZero), + (0, 3, 1, "vae1is", Xt::Always), + (0, 3, 2, "aside1is", Xt::Always), + (0, 3, 3, "vaae1is", Xt::Always), + (0, 3, 5, "vale1is", Xt::Always), + (0, 3, 7, "vaale1is", Xt::Always), + (0, 5, 1, "rvae1os", Xt::Always), + (0, 5, 3, "rvaae1os", Xt::Always), + (0, 5, 5, "rvale1os", Xt::Always), + (0, 5, 7, "rvaale1os", Xt::Always), + (0, 6, 1, "rvae1", Xt::Always), + (0, 6, 3, "rvaae1", Xt::Always), + (0, 6, 5, "rvale1", Xt::Always), + (0, 6, 7, "rvaale1", Xt::Always), + (0, 7, 0, "vmalle1", Xt::Never), + (0, 7, 1, "vae1", Xt::Always), + (0, 7, 2, "aside1", Xt::Always), + (0, 7, 3, "vaae1", Xt::Always), + (0, 7, 5, "vale1", Xt::Always), + (0, 7, 7, "vaale1", Xt::Always), + (4, 0, 1, "ipas2e1is", Xt::Always), + (4, 0, 2, "ripas2e1is", Xt::Always), + (4, 0, 5, "ipas2le1is", Xt::Always), + (4, 0, 6, "ripas2le1is", Xt::Always), + (4, 1, 0, "alle2os", Xt::UnlessZero), + (4, 1, 1, "vae2os", Xt::Always), + (4, 1, 4, "alle1os", Xt::UnlessZero), + (4, 1, 5, "vale2os", Xt::Always), + (4, 1, 6, "vmalls12e1os", Xt::UnlessZero), + (4, 2, 1, "rvae2is", Xt::Always), + (4, 2, 2, "vmallws2e1is", Xt::UnlessZero), + (4, 2, 5, "rvale2is", Xt::Always), + (4, 3, 0, "alle2is", Xt::UnlessZero), + (4, 3, 1, "vae2is", Xt::Always), + (4, 3, 4, "alle1is", Xt::UnlessZero), + (4, 3, 5, "vale2is", Xt::Always), + (4, 3, 6, "vmalls12e1is", Xt::UnlessZero), + (4, 4, 0, "ipas2e1os", Xt::Always), + (4, 4, 1, "ipas2e1", Xt::Always), + (4, 4, 2, "ripas2e1", Xt::Always), + (4, 4, 3, "ripas2e1os", Xt::Always), + (4, 4, 4, "ipas2le1os", Xt::Always), + (4, 4, 5, "ipas2le1", Xt::Always), + (4, 4, 6, "ripas2le1", Xt::Always), + (4, 4, 7, "ripas2le1os", Xt::Always), + (4, 5, 1, "rvae2os", Xt::Always), + (4, 5, 2, "vmallws2e1os", Xt::UnlessZero), + (4, 5, 5, "rvale2os", Xt::Always), + (4, 6, 1, "rvae2", Xt::Always), + (4, 6, 3, "vmallws2e1", Xt::Never), + (4, 6, 5, "rvale2", Xt::Always), + (4, 7, 0, "alle2", Xt::Never), + (4, 7, 1, "vae2", Xt::Always), + (4, 7, 4, "alle1", Xt::Never), + (4, 7, 5, "vale2", Xt::Always), + (4, 7, 6, "vmalls12e1", Xt::Never), + (6, 1, 0, "alle3os", Xt::Never), + (6, 1, 1, "vae3os", Xt::Always), + (6, 1, 4, "paallos", Xt::Never), + (6, 1, 5, "vale3os", Xt::Always), + (6, 2, 1, "rvae3is", Xt::Always), + (6, 2, 5, "rvale3is", Xt::Always), + (6, 3, 0, "alle3is", Xt::Never), + (6, 3, 1, "vae3is", Xt::Always), + (6, 3, 5, "vale3is", Xt::Always), + (6, 4, 3, "rpaos", Xt::Always), + (6, 4, 7, "rpalos", Xt::Always), + (6, 5, 1, "rvae3os", Xt::Always), + (6, 5, 5, "rvale3os", Xt::Always), + (6, 6, 1, "rvae3", Xt::Always), + (6, 6, 5, "rvale3", Xt::Always), + (6, 7, 0, "alle3", Xt::Never), + (6, 7, 1, "vae3", Xt::Always), + (6, 7, 4, "paall", Xt::Never), + (6, 7, 5, "vale3", Xt::Always), +]; + +fn lookup(table: &[(u8, u8, u8, &'static str, Xt)], op: &SysOp) -> Option<(&'static str, Xt)> { + table + .iter() + .find(|&&(op1, crm, op2, _, _)| (op1, crm, op2) == (op.op1, op.crm, op.op2)) + .map(|&(_, _, _, name, xt)| (name, xt)) +} + +/// `mnemonic name{, xt}` for a named alias, or the generic `sys` form. +fn named_sys( + f: &mut Formatter<'_>, + mnemonic: &str, + named: Option<(&str, Xt)>, + op: &SysOp, +) -> fmt::Result { + let Some((name, xt)) = named else { + return generic_sys(f, op); + }; + write!(f, "{mnemonic} {name}")?; + match xt { + Xt::Never => Ok(()), + Xt::UnlessZero if op.rt == 31 => Ok(()), + Xt::Always | Xt::UnlessZero => { + f.write_str(", ")?; + gpr(f, Width::X64, op.rt) + } + } +} + +fn generic_sys(f: &mut Formatter<'_>, op: &SysOp) -> fmt::Result { + write!(f, "sys #{}, C{}, C{}, #{}", op.op1, op.crn, op.crm, op.op2)?; + if op.rt != 31 { + f.write_str(", ")?; + gpr(f, Width::X64, op.rt)?; + } + Ok(()) +} + +fn cache_op(f: &mut Formatter<'_>, op: &SysOp) -> fmt::Result { + match lookup(IC, op) { + Some(ic) => named_sys(f, "ic", Some(ic), op), + None => named_sys(f, "dc", lookup(DC, op), op), + } +} + +fn tlbi(f: &mut Formatter<'_>, op: &SysOp) -> fmt::Result { + named_sys(f, "tlbi", lookup(TLBI, op), op) +} + +/// `(op1, crm, op2)` of the Apple IMP-DEF register in `System::apple[slot]` +/// (`S3__C15_C_`). +const APPLE: [(u8, u8, u8); 21] = [ + (1, 0, 0), + (1, 1, 0), + (1, 2, 0), + (1, 3, 0), + (1, 4, 0), + (1, 5, 0), + (1, 6, 0), + (2, 0, 0), + (2, 1, 0), + (2, 2, 0), + (2, 3, 0), + (2, 4, 0), + (2, 5, 0), + (2, 6, 0), + (2, 7, 0), + (2, 9, 0), + (2, 10, 0), + (5, 0, 0), + (5, 0, 1), + (5, 3, 1), + (7, 0, 4), +]; + +fn generic_register( + f: &mut Formatter<'_>, + op0: u8, + op1: u8, + crn: u8, + crm: u8, + op2: u8, +) -> fmt::Result { + write!(f, "s{op0}_{op1}_c{crn}_c{crm}_{op2}") +} + +fn register_name(f: &mut Formatter<'_>, register: SystemRegister) -> fmt::Result { + use SystemRegister as R; + let name = match register { + R::SpEl0 => "sp_el0", + R::SpEl1 => "sp_el1", + R::SpEl2 => "sp_el2", + R::SctlrEl2 => "sctlr_el2", + R::TcrEl2 => "tcr_el2", + R::Ttbr0El2 => "ttbr0_el2", + R::Ttbr1El2 => "ttbr1_el2", + R::MairEl2 => "mair_el2", + R::VbarEl2 => "vbar_el2", + R::ElrEl2 => "elr_el2", + R::SpsrEl2 => "spsr_el2", + R::EsrEl2 => "esr_el2", + R::FarEl2 => "far_el2", + R::HcrEl2 => "hcr_el2", + R::TpidrEl2 => "tpidr_el2", + R::CntvoffEl2 => "cntvoff_el2", + R::CurrentEl => "currentel", + R::SpsrEl1 => "spsr_el1", + R::ElrEl1 => "elr_el1", + R::EsrEl1 => "esr_el1", + R::FarEl1 => "far_el1", + R::ParEl1 => "par_el1", + R::VbarEl1 => "vbar_el1", + R::CpacrEl1 => "cpacr_el1", + R::SctlrEl1 => "sctlr_el1", + R::Ttbr0El1 => "ttbr0_el1", + R::Ttbr1El1 => "ttbr1_el1", + R::TcrEl1 => "tcr_el1", + R::MairEl1 => "mair_el1", + R::TpidrEl0 => "tpidr_el0", + R::TpidrroEl0 => "tpidrro_el0", + R::CntvctEl0 => "cntvct_el0", + R::CntvCtlEl0 => "cntv_ctl_el0", + R::CntvCvalEl0 => "cntv_cval_el0", + R::CntvTvalEl0 => "cntv_tval_el0", + R::TpidrEl1 => "tpidr_el1", + R::ContextidrEl1 => "contextidr_el1", + R::MdscrEl1 => "mdscr_el1", + R::OslarEl1 => "oslar_el1", + R::OsdlrEl1 => "osdlr_el1", + R::OslsrEl1 => "oslsr_el1", + R::CntkctlEl1 => "cntkctl_el1", + R::Fpcr => "fpcr", + R::Fpsr => "fpsr", + R::Daif => "daif", + R::SpSel => "spsel", + R::Pan => "pan", + R::IsrEl1 => "isr_el1", + R::IccSreEl1 => "icc_sre_el1", + R::IccPmrEl1 => "icc_pmr_el1", + R::IccBpr0El1 => "icc_bpr0_el1", + R::IccCtlrEl1 => "icc_ctlr_el1", + R::IccIgrpen0El1 => "icc_igrpen0_el1", + R::IccIgrpen1El1 => "icc_igrpen1_el1", + R::IccBpr1El1 => "icc_bpr1_el1", + R::CntpCtlEl0 => "cntp_ctl_el0", + R::CntpctEl0 => "cntpct_el0", + R::CntpTvalEl0 => "cntp_tval_el0", + R::CntpCvalEl0 => "cntp_cval_el0", + R::IccIar1El1 => "icc_iar1_el1", + R::IccEoir1El1 => "icc_eoir1_el1", + R::ActlrEl1 => "actlr_el1", + R::MidrEl1 => "midr_el1", + R::MpidrEl1 => "mpidr_el1", + R::RevidrEl1 => "revidr_el1", + R::CtrEl0 => "ctr_el0", + R::DczidEl0 => "dczid_el0", + R::CntfrqEl0 => "cntfrq_el0", + R::ClidrEl1 => "clidr_el1", + R::CcsidrEl1 => "ccsidr_el1", + R::CsselrEl1 => "csselr_el1", + R::IdAa64pfr0El1 => "id_aa64pfr0_el1", + R::IdAa64pfr1El1 => "id_aa64pfr1_el1", + R::IdAa64pfr2El1 => "id_aa64pfr2_el1", + R::IdAa64zfr0El1 => "id_aa64zfr0_el1", + R::IdAa64smfr0El1 => "id_aa64smfr0_el1", + R::IdAa64fpfr0El1 => "id_aa64fpfr0_el1", + R::IdAa64isar3El1 => "id_aa64isar3_el1", + R::AidrEl1 => "aidr_el1", + R::IdAa64dfr0El1 => "id_aa64dfr0_el1", + R::IdAa64dfr1El1 => "id_aa64dfr1_el1", + R::IdAa64isar0El1 => "id_aa64isar0_el1", + R::IdAa64isar1El1 => "id_aa64isar1_el1", + R::IdAa64isar2El1 => "id_aa64isar2_el1", + R::IdAa64mmfr0El1 => "id_aa64mmfr0_el1", + R::IdAa64mmfr1El1 => "id_aa64mmfr1_el1", + R::IdAa64mmfr2El1 => "id_aa64mmfr2_el1", + R::IdAa64mmfr3El1 => "id_aa64mmfr3_el1", + R::IdAa64mmfr4El1 => "id_aa64mmfr4_el1", + R::Nzcv => "nzcv", + R::DaifSet => "daifset", + R::DaifClear => "daifclr", + R::SpSelSet => "spsel", + R::PanSet => "pan", + R::Apple(slot) => { + let (op1, crm, op2) = APPLE[usize::from(slot) % APPLE.len()]; + return generic_register(f, 3, op1, 15, crm, op2); + } + R::AppleBank(slot) => { + let op1 = ((slot >> 7) & 7) as u8; + let crm = ((slot >> 3) & 15) as u8; + let op2 = (slot & 7) as u8; + return generic_register(f, 3, op1, 15, crm, op2); + } + R::AppleZero { op1, crm, op2 } => return generic_register(f, 3, op1, 15, crm, op2), + R::RazWi { crm, op2 } => { + let name = ["dbgbvr", "dbgbcr", "dbgwvr", "dbgwcr"][usize::from(op2 & 3)]; + return write!(f, "{name}{crm}_el1"); + } + R::PacKey(slot) => { + let (key, half) = [ + ("apia", "lo"), + ("apia", "hi"), + ("apib", "lo"), + ("apib", "hi"), + ("apda", "lo"), + ("apda", "hi"), + ("apdb", "lo"), + ("apdb", "hi"), + ("apga", "lo"), + ("apga", "hi"), + ][usize::from(slot) % 10]; + return write!(f, "{key}key{half}_el1"); + } + }; + f.write_str(name) +} + +fn system_register_move(f: &mut Formatter<'_>, system: &decode::System) -> fmt::Result { + use SystemRegister as R; + if system.read { + f.write_str("mrs ")?; + gpr(f, Width::X64, system.rt)?; + f.write_str(", ")?; + return register_name(f, system.register); + } + f.write_str("msr ")?; + register_name(f, system.register)?; + f.write_str(", ")?; + match system.register { + R::DaifSet | R::DaifClear | R::SpSelSet | R::PanSet => { + write!(f, "#{:#x}", system.immediate) + } + _ => gpr(f, Width::X64, system.rt), + } +} + +/// The key letter of `braa`/`retab`/`eretaa`: `a` then `a` or `b`. +fn branch_key(key: KeyId) -> &'static str { + match key { + KeyId::Ib => "ab", + _ => "aa", + } +} + +/// The key as `` (`ga` for the generic key); with a zero modifier the +/// `z` goes between the two letters (`paciza`, `autdzb`). +fn key_name(f: &mut Formatter<'_>, key: KeyId, zero: bool) -> fmt::Result { + let (kind, which) = match key { + KeyId::Ia => ("i", "a"), + KeyId::Ib => ("i", "b"), + KeyId::Da => ("d", "a"), + KeyId::Db => ("d", "b"), + KeyId::Ga => ("g", "a"), + }; + write!(f, "{kind}{}{which}", if zero { "z" } else { "" }) +} + +/// `sp` for register 31, else `xN`: the modifier register of pointer authentication. +fn modifier_register(f: &mut Formatter<'_>, r: u8) -> fmt::Result { + if r == 31 { + f.write_str("sp") + } else { + write!(f, "x{r}") + } +} + +fn pauth_op(f: &mut Formatter<'_>, op: pauth::Op) -> fmt::Result { + use pauth::Op::{Auth, Branch, ExceptionReturn, Pacga, Return, Sign, Strip}; + match op { + Sign { + key, + reg, + modifier, + hint, + } + | Auth { + key, + reg, + modifier, + hint, + } => { + f.write_str(if matches!(op, Sign { .. }) { + "pac" + } else { + "aut" + })?; + if hint { + // `paciaz`, `paciasp`, `pacia1716`: no operands. + key_name(f, key, false)?; + return f.write_str(match modifier { + Modifier::Zero => "z", + Modifier::Reg(31) => "sp", + Modifier::Reg(_) => "1716", + }); + } + key_name(f, key, matches!(modifier, Modifier::Zero))?; + f.write_str(" ")?; + gpr(f, Width::X64, reg)?; + if let Modifier::Reg(m) = modifier { + f.write_str(", ")?; + modifier_register(f, m)?; + } + Ok(()) + } + Strip { hint: true, .. } => f.write_str("xpaclri"), + Strip { data, reg, .. } => { + f.write_str(if data { "xpacd " } else { "xpaci " })?; + gpr(f, Width::X64, reg) + } + Pacga { rd, rn, modifier } => { + f.write_str("pacga ")?; + gpr(f, Width::X64, rd)?; + f.write_str(", ")?; + gpr(f, Width::X64, rn)?; + f.write_str(", ")?; + match modifier { + Modifier::Reg(m) => modifier_register(f, m), + Modifier::Zero => f.write_str("xzr"), + } + } + Branch { + key, + link, + target, + modifier, + } => { + write!(f, "{}{}", if link { "blr" } else { "br" }, branch_key(key))?; + if matches!(modifier, Modifier::Zero) { + f.write_str("z")?; + } + f.write_str(" ")?; + gpr(f, Width::X64, target)?; + if let Modifier::Reg(m) = modifier { + f.write_str(", ")?; + modifier_register(f, m)?; + } + Ok(()) + } + Return { key } => write!(f, "ret{}", branch_key(key)), + ExceptionReturn { key } => write!(f, "eret{}", branch_key(key)), + } +} + +#[cfg(test)] +mod tests { + use crate::format::{word, word_at}; + + fn check(w: u32, expected: &str) { + assert_eq!( + std::string::ToString::to_string(&word(w)), + expected, + "{w:#010x}" + ); + } + + fn check_at(w: u32, pc: u64, expected: &str) { + assert_eq!( + std::string::ToString::to_string(&word_at(w, pc)), + expected, + "{w:#010x} at {pc:#x}" + ); + } + + #[test] + fn relative_branches_without_an_address() { + check(0x14000002, "b .+8"); + check(0x17ffffff, "b .-4"); + check(0x94000040, "bl .+256"); + check(0x97fffffe, "bl .-8"); + check(0x54000040, "b.eq .+8"); + check(0x54ffffe1, "b.ne .-4"); + check(0x34000040, "cbz w0, .+8"); + check(0xb5fffff5, "cbnz x21, .-4"); + check(0x36180040, "tbz w0, #3, .+8"); + check(0xb7f80041, "tbnz x1, #63, .+8"); + } + + #[test] + fn relative_branches_at_an_address() { + check_at(0x14000002, 0x10100000, "b 0x10100008"); + check_at(0x94000040, 0x10100004, "bl 0x10100104"); + check_at(0x17ffffff, 0x10100008, "b 0x10100004"); + check_at(0x97fffffe, 0x1010000c, "bl 0x10100004"); + check_at(0x15ffffff, 0x10100010, "b 0x1810000c"); + check_at(0x96000000, 0x10100014, "bl 0x8100014"); + check_at(0x54000040, 0x10100018, "b.eq 0x10100020"); + check_at(0x54ffffe1, 0x1010001c, "b.ne 0x10100018"); + check_at(0x543fffe2, 0x10100020, "b.cs 0x1018001c"); + check_at(0x54800003, 0x10100024, "b.cc 0x10000024"); + check_at(0x54000024, 0x10100028, "b.mi 0x1010002c"); + check_at(0x54000025, 0x1010002c, "b.pl 0x10100030"); + check_at(0x54000026, 0x10100030, "b.vs 0x10100034"); + check_at(0x54000027, 0x10100034, "b.vc 0x10100038"); + check_at(0x54000028, 0x10100038, "b.hi 0x1010003c"); + check_at(0x54000029, 0x1010003c, "b.ls 0x10100040"); + check_at(0x5400002a, 0x10100040, "b.ge 0x10100044"); + check_at(0x5400002b, 0x10100044, "b.lt 0x10100048"); + check_at(0x5400002c, 0x10100048, "b.gt 0x1010004c"); + check_at(0x5400002d, 0x1010004c, "b.le 0x10100050"); + check_at(0x34000040, 0x10100050, "cbz w0, 0x10100058"); + check_at(0x35ffffc5, 0x10100054, "cbnz w5, 0x1010004c"); + check_at(0xb400009f, 0x10100058, "cbz xzr, 0x10100068"); + check_at(0xb580001e, 0x1010005c, "cbnz x30, 0x1000005c"); + check_at(0x357fffff, 0x10100060, "cbnz wzr, 0x1020005c"); + check_at(0x36000040, 0x10100064, "tbz w0, #0, 0x1010006c"); + check_at(0x372fffc3, 0x10100068, "tbnz w3, #5, 0x10100060"); + check_at(0x36f8ffff, 0x1010006c, "tbz wzr, #31, 0x10102068"); + check_at(0xb704001e, 0x10100070, "tbnz x30, #32, 0x100f8070"); + check_at(0xb6f80205, 0x10100074, "tbz x5, #63, 0x101000b4"); + check_at(0xb7080020, 0x10100078, "tbnz x0, #33, 0x1010007c"); + } + + #[test] + fn hints() { + check(0xd503201f, "nop"); + check(0xd503203f, "yield"); + check(0xd503205f, "wfe"); + check(0xd503207f, "wfi"); + check(0xd503209f, "sev"); + check(0xd50320bf, "sevl"); + check(0xd50320df, "dgh"); + check(0xd50320ff, "xpaclri"); + check(0xd503211f, "pacia1716"); + check(0xd503215f, "pacib1716"); + check(0xd503219f, "autia1716"); + check(0xd50321df, "autib1716"); + check(0xd50321ff, "hint #0xf"); + check(0xd503221f, "esb"); + check(0xd503223f, "psb csync"); + check(0xd503225f, "tsb csync"); + check(0xd503227f, "gcsb dsync"); + check(0xd503229f, "csdb"); + check(0xd50322bf, "hint #0x15"); + check(0xd50322df, "clrbhb"); + check(0xd50322ff, "hint #0x17"); + check(0xd503231f, "paciaz"); + check(0xd503233f, "paciasp"); + check(0xd503235f, "pacibz"); + check(0xd503237f, "pacibsp"); + check(0xd503239f, "autiaz"); + check(0xd50323bf, "autiasp"); + check(0xd50323df, "autibz"); + check(0xd50323ff, "autibsp"); + check(0xd503241f, "bti r"); + check(0xd503243f, "hint #0x21"); + check(0xd503245f, "bti c"); + check(0xd503249f, "bti j"); + check(0xd50324df, "bti jc"); + check(0xd503251f, "chkfeat x16"); + check(0xd503261f, "stshh keep"); + check(0xd503263f, "stshh strm"); + check(0xd503265f, "shuh"); + check(0xd503267f, "shuh ph"); + check(0xd503269f, "stcph"); + check(0xd50326bf, "hint #0x35"); + check(0xd5032fff, "hint #0x7f"); + } + + #[test] + fn barriers_and_clrex() { + check(0xd503305f, "clrex #0x0"); + check(0xd503315f, "clrex #0x1"); + check(0xd503325f, "clrex #0x2"); + check(0xd503335f, "clrex #0x3"); + check(0xd503345f, "clrex #0x4"); + check(0xd503355f, "clrex #0x5"); + check(0xd503365f, "clrex #0x6"); + check(0xd503375f, "clrex #0x7"); + check(0xd503385f, "clrex #0x8"); + check(0xd503395f, "clrex #0x9"); + check(0xd5033a5f, "clrex #0xa"); + check(0xd5033b5f, "clrex #0xb"); + check(0xd5033c5f, "clrex #0xc"); + check(0xd5033d5f, "clrex #0xd"); + check(0xd5033e5f, "clrex #0xe"); + check(0xd5033f5f, "clrex"); + check(0xd503309f, "ssbb"); + check(0xd503319f, "dsb oshld"); + check(0xd503329f, "dsb oshst"); + check(0xd503339f, "dsb osh"); + check(0xd503349f, "pssbb"); + check(0xd503359f, "dsb nshld"); + check(0xd503369f, "dsb nshst"); + check(0xd503379f, "dsb nsh"); + check(0xd503389f, "dsb #0x08"); + check(0xd503399f, "dsb ishld"); + check(0xd5033a9f, "dsb ishst"); + check(0xd5033b9f, "dsb ish"); + check(0xd5033c9f, "dsb #0x0c"); + check(0xd5033d9f, "dsb ld"); + check(0xd5033e9f, "dsb st"); + check(0xd5033f9f, "dsb sy"); + check(0xd50330bf, "dmb #0x00"); + check(0xd50331bf, "dmb oshld"); + check(0xd50332bf, "dmb oshst"); + check(0xd50333bf, "dmb osh"); + check(0xd50334bf, "dmb #0x04"); + check(0xd50335bf, "dmb nshld"); + check(0xd50336bf, "dmb nshst"); + check(0xd50337bf, "dmb nsh"); + check(0xd50338bf, "dmb #0x08"); + check(0xd50339bf, "dmb ishld"); + check(0xd5033abf, "dmb ishst"); + check(0xd5033bbf, "dmb ish"); + check(0xd5033cbf, "dmb #0x0c"); + check(0xd5033dbf, "dmb ld"); + check(0xd5033ebf, "dmb st"); + check(0xd5033fbf, "dmb sy"); + check(0xd50330df, "isb #0x0"); + check(0xd50331df, "isb #0x1"); + check(0xd50332df, "isb #0x2"); + check(0xd50333df, "isb #0x3"); + check(0xd50334df, "isb #0x4"); + check(0xd50335df, "isb #0x5"); + check(0xd50336df, "isb #0x6"); + check(0xd50337df, "isb #0x7"); + check(0xd50338df, "isb #0x8"); + check(0xd50339df, "isb #0x9"); + check(0xd5033adf, "isb #0xa"); + check(0xd5033bdf, "isb #0xb"); + check(0xd5033cdf, "isb #0xc"); + check(0xd5033ddf, "isb #0xd"); + check(0xd5033edf, "isb #0xe"); + check(0xd5033fdf, "isb"); + } + + #[test] + fn every_system_register_reads() { + check(0xd5384100, "mrs x0, sp_el0"); + check(0xd53c4100, "mrs x0, sp_el1"); + check(0xd53e4100, "mrs x0, sp_el2"); + check(0xd53c1000, "mrs x0, sctlr_el2"); + check(0xd53c2040, "mrs x0, tcr_el2"); + check(0xd53c2000, "mrs x0, ttbr0_el2"); + check(0xd53c2020, "mrs x0, ttbr1_el2"); + check(0xd53ca200, "mrs x0, mair_el2"); + check(0xd53cc000, "mrs x0, vbar_el2"); + check(0xd53c4020, "mrs x0, elr_el2"); + check(0xd53c4000, "mrs x0, spsr_el2"); + check(0xd53c5200, "mrs x0, esr_el2"); + check(0xd53c6000, "mrs x0, far_el2"); + check(0xd53c1100, "mrs x0, hcr_el2"); + check(0xd53cd040, "mrs x0, tpidr_el2"); + check(0xd53ce060, "mrs x0, cntvoff_el2"); + check(0xd5384240, "mrs x0, currentel"); + check(0xd5384000, "mrs x0, spsr_el1"); + check(0xd5384020, "mrs x0, elr_el1"); + check(0xd5385200, "mrs x0, esr_el1"); + check(0xd5386000, "mrs x0, far_el1"); + check(0xd5387400, "mrs x0, par_el1"); + check(0xd538c000, "mrs x0, vbar_el1"); + check(0xd5381040, "mrs x0, cpacr_el1"); + check(0xd5381000, "mrs x0, sctlr_el1"); + check(0xd5382000, "mrs x0, ttbr0_el1"); + check(0xd5382020, "mrs x0, ttbr1_el1"); + check(0xd5382040, "mrs x0, tcr_el1"); + check(0xd538a200, "mrs x0, mair_el1"); + check(0xd53bd040, "mrs x0, tpidr_el0"); + check(0xd53bd060, "mrs x0, tpidrro_el0"); + check(0xd53be040, "mrs x0, cntvct_el0"); + check(0xd53be320, "mrs x0, cntv_ctl_el0"); + check(0xd53be340, "mrs x0, cntv_cval_el0"); + check(0xd53be300, "mrs x0, cntv_tval_el0"); + check(0xd538d080, "mrs x0, tpidr_el1"); + check(0xd538d020, "mrs x0, contextidr_el1"); + check(0xd5300240, "mrs x0, mdscr_el1"); + check(0xd5301080, "mrs x0, oslar_el1"); + check(0xd5301380, "mrs x0, osdlr_el1"); + check(0xd5301180, "mrs x0, oslsr_el1"); + check(0xd538e100, "mrs x0, cntkctl_el1"); + check(0xd53b4400, "mrs x0, fpcr"); + check(0xd53b4420, "mrs x0, fpsr"); + check(0xd53b4220, "mrs x0, daif"); + check(0xd5384200, "mrs x0, spsel"); + check(0xd5384260, "mrs x0, pan"); + check(0xd538c100, "mrs x0, isr_el1"); + check(0xd538cca0, "mrs x0, icc_sre_el1"); + check(0xd5384600, "mrs x0, icc_pmr_el1"); + check(0xd538c860, "mrs x0, icc_bpr0_el1"); + check(0xd538cc80, "mrs x0, icc_ctlr_el1"); + check(0xd538ccc0, "mrs x0, icc_igrpen0_el1"); + check(0xd538cce0, "mrs x0, icc_igrpen1_el1"); + check(0xd538cc60, "mrs x0, icc_bpr1_el1"); + check(0xd53be220, "mrs x0, cntp_ctl_el0"); + check(0xd53be020, "mrs x0, cntpct_el0"); + check(0xd53be200, "mrs x0, cntp_tval_el0"); + check(0xd53be240, "mrs x0, cntp_cval_el0"); + check(0xd538cc00, "mrs x0, icc_iar1_el1"); + check(0xd538cc20, "mrs x0, icc_eoir1_el1"); + check(0xd5381020, "mrs x0, actlr_el1"); + check(0xd5380000, "mrs x0, midr_el1"); + check(0xd53800a0, "mrs x0, mpidr_el1"); + check(0xd53800c0, "mrs x0, revidr_el1"); + check(0xd53b0020, "mrs x0, ctr_el0"); + check(0xd53b00e0, "mrs x0, dczid_el0"); + check(0xd53be000, "mrs x0, cntfrq_el0"); + check(0xd5390020, "mrs x0, clidr_el1"); + check(0xd5390000, "mrs x0, ccsidr_el1"); + check(0xd53a0000, "mrs x0, csselr_el1"); + check(0xd5380400, "mrs x0, id_aa64pfr0_el1"); + check(0xd5380420, "mrs x0, id_aa64pfr1_el1"); + check(0xd5380440, "mrs x0, id_aa64pfr2_el1"); + check(0xd5380480, "mrs x0, id_aa64zfr0_el1"); + check(0xd53804a0, "mrs x0, id_aa64smfr0_el1"); + check(0xd53804e0, "mrs x0, id_aa64fpfr0_el1"); + check(0xd5380660, "mrs x0, id_aa64isar3_el1"); + check(0xd53900e0, "mrs x0, aidr_el1"); + check(0xd5380500, "mrs x0, id_aa64dfr0_el1"); + check(0xd5380520, "mrs x0, id_aa64dfr1_el1"); + check(0xd5380600, "mrs x0, id_aa64isar0_el1"); + check(0xd5380620, "mrs x0, id_aa64isar1_el1"); + check(0xd5380640, "mrs x0, id_aa64isar2_el1"); + check(0xd5380700, "mrs x0, id_aa64mmfr0_el1"); + check(0xd5380720, "mrs x0, id_aa64mmfr1_el1"); + check(0xd5380740, "mrs x0, id_aa64mmfr2_el1"); + check(0xd5380760, "mrs x0, id_aa64mmfr3_el1"); + check(0xd5380780, "mrs x0, id_aa64mmfr4_el1"); + check(0xd53b4200, "mrs x0, nzcv"); + } + + #[test] + fn every_system_register_writes() { + check(0xd5184100, "msr sp_el0, x0"); + check(0xd51c4100, "msr sp_el1, x0"); + check(0xd51e4100, "msr sp_el2, x0"); + check(0xd51c1000, "msr sctlr_el2, x0"); + check(0xd51c2040, "msr tcr_el2, x0"); + check(0xd51c2000, "msr ttbr0_el2, x0"); + check(0xd51c2020, "msr ttbr1_el2, x0"); + check(0xd51ca200, "msr mair_el2, x0"); + check(0xd51cc000, "msr vbar_el2, x0"); + check(0xd51c4020, "msr elr_el2, x0"); + check(0xd51c4000, "msr spsr_el2, x0"); + check(0xd51c5200, "msr esr_el2, x0"); + check(0xd51c6000, "msr far_el2, x0"); + check(0xd51c1100, "msr hcr_el2, x0"); + check(0xd51cd040, "msr tpidr_el2, x0"); + check(0xd51ce060, "msr cntvoff_el2, x0"); + check(0xd5184000, "msr spsr_el1, x0"); + check(0xd5184020, "msr elr_el1, x0"); + check(0xd5185200, "msr esr_el1, x0"); + check(0xd5186000, "msr far_el1, x0"); + check(0xd5187400, "msr par_el1, x0"); + check(0xd518c000, "msr vbar_el1, x0"); + check(0xd5181040, "msr cpacr_el1, x0"); + check(0xd5181000, "msr sctlr_el1, x0"); + check(0xd5182000, "msr ttbr0_el1, x0"); + check(0xd5182020, "msr ttbr1_el1, x0"); + check(0xd5182040, "msr tcr_el1, x0"); + check(0xd518a200, "msr mair_el1, x0"); + check(0xd51bd040, "msr tpidr_el0, x0"); + check(0xd51bd060, "msr tpidrro_el0, x0"); + check(0xd51be040, "msr cntvct_el0, x0"); + check(0xd51be320, "msr cntv_ctl_el0, x0"); + check(0xd51be340, "msr cntv_cval_el0, x0"); + check(0xd51be300, "msr cntv_tval_el0, x0"); + check(0xd518d080, "msr tpidr_el1, x0"); + check(0xd518d020, "msr contextidr_el1, x0"); + check(0xd5100240, "msr mdscr_el1, x0"); + check(0xd5101080, "msr oslar_el1, x0"); + check(0xd5101380, "msr osdlr_el1, x0"); + check(0xd5101180, "msr oslsr_el1, x0"); + check(0xd518e100, "msr cntkctl_el1, x0"); + check(0xd51b4400, "msr fpcr, x0"); + check(0xd51b4420, "msr fpsr, x0"); + check(0xd51b4220, "msr daif, x0"); + check(0xd5184200, "msr spsel, x0"); + check(0xd5184260, "msr pan, x0"); + check(0xd518cca0, "msr icc_sre_el1, x0"); + check(0xd5184600, "msr icc_pmr_el1, x0"); + check(0xd518c860, "msr icc_bpr0_el1, x0"); + check(0xd518cc80, "msr icc_ctlr_el1, x0"); + check(0xd518ccc0, "msr icc_igrpen0_el1, x0"); + check(0xd518cce0, "msr icc_igrpen1_el1, x0"); + check(0xd518cc60, "msr icc_bpr1_el1, x0"); + check(0xd51be220, "msr cntp_ctl_el0, x0"); + check(0xd51be020, "msr cntpct_el0, x0"); + check(0xd51be200, "msr cntp_tval_el0, x0"); + check(0xd51be240, "msr cntp_cval_el0, x0"); + check(0xd518cc00, "msr icc_iar1_el1, x0"); + check(0xd518cc20, "msr icc_eoir1_el1, x0"); + check(0xd5181020, "msr actlr_el1, x0"); + check(0xd51a0000, "msr csselr_el1, x0"); + check(0xd51b4200, "msr nzcv, x0"); + } + + #[test] + fn system_register_edge_cases() { + check(0xd5034fdf, "msr daifset, #0xf"); + check(0xd50342df, "msr daifset, #0x2"); + check(0xd5034cdf, "msr daifset, #0xc"); + check(0xd5034fff, "msr daifclr, #0xf"); + check(0xd50342ff, "msr daifclr, #0x2"); + check(0xd500409f, "msr pan, #0x0"); + check(0xd500419f, "msr pan, #0x1"); + check(0xd50040bf, "msr spsel, #0x0"); + check(0xd50041bf, "msr spsel, #0x1"); + check(0xd5382100, "mrs x0, apiakeylo_el1"); + check(0xd5382120, "mrs x0, apiakeyhi_el1"); + check(0xd5382140, "mrs x0, apibkeylo_el1"); + check(0xd5382160, "mrs x0, apibkeyhi_el1"); + check(0xd5382200, "mrs x0, apdakeylo_el1"); + check(0xd5382300, "mrs x0, apgakeylo_el1"); + check(0xd5382320, "mrs x0, apgakeyhi_el1"); + } + + #[test] + fn cache_maintenance() { + check(0xd5087100, "ic ialluis"); + check(0xd5087500, "ic iallu"); + check(0xd5087620, "dc ivac, x0"); + check(0xd5087640, "dc isw, x0"); + check(0xd5087660, "dc igvac, x0"); + check(0xd5087680, "dc igsw, x0"); + check(0xd50876a0, "dc igdvac, x0"); + check(0xd50876c0, "dc igdsw, x0"); + check(0xd5087a40, "dc csw, x0"); + check(0xd5087a80, "dc cgsw, x0"); + check(0xd5087ac0, "dc cgdsw, x0"); + check(0xd5087e40, "dc cisw, x0"); + check(0xd5087e80, "dc cigsw, x0"); + check(0xd5087ec0, "dc cigdsw, x0"); + check(0xd50b7520, "ic ivau, x0"); + check(0xd50b7a20, "dc cvac, x0"); + check(0xd50b7a60, "dc cgvac, x0"); + check(0xd50b7aa0, "dc cgdvac, x0"); + check(0xd50b7b00, "dc cvaoc, x0"); + check(0xd50b7b20, "dc cvau, x0"); + check(0xd50b7be0, "dc cgdvaoc, x0"); + check(0xd50b7e20, "dc civac, x0"); + check(0xd50b7e60, "dc cigvac, x0"); + check(0xd50b7ea0, "dc cigdvac, x0"); + check(0xd50c7e00, "dc cipae, x0"); + check(0xd50c7ee0, "dc cigdpae, x0"); + check(0xd50e7e20, "dc cipapa, x0"); + check(0xd50e7ea0, "dc cigdpapa, x0"); + check(0xd508711f, "ic ialluis"); + check(0xd508751f, "ic iallu"); + check(0xd508763f, "dc ivac, xzr"); + check(0xd508765f, "dc isw, xzr"); + check(0xd508767f, "dc igvac, xzr"); + check(0xd508769f, "dc igsw, xzr"); + check(0xd50876bf, "dc igdvac, xzr"); + check(0xd50876df, "dc igdsw, xzr"); + check(0xd5087a5f, "dc csw, xzr"); + check(0xd5087a9f, "dc cgsw, xzr"); + check(0xd5087adf, "dc cgdsw, xzr"); + check(0xd5087e5f, "dc cisw, xzr"); + check(0xd5087e9f, "dc cigsw, xzr"); + check(0xd5087edf, "dc cigdsw, xzr"); + check(0xd50b753f, "ic ivau, xzr"); + check(0xd50b7a3f, "dc cvac, xzr"); + check(0xd50b7a7f, "dc cgvac, xzr"); + check(0xd50b7abf, "dc cgdvac, xzr"); + check(0xd50b7b1f, "dc cvaoc, xzr"); + check(0xd50b7b3f, "dc cvau, xzr"); + check(0xd50b7bff, "dc cgdvaoc, xzr"); + check(0xd50b7e3f, "dc civac, xzr"); + check(0xd50b7e7f, "dc cigvac, xzr"); + check(0xd50b7ebf, "dc cigdvac, xzr"); + check(0xd50c7e1f, "dc cipae, xzr"); + check(0xd50c7eff, "dc cigdpae, xzr"); + check(0xd50e7e3f, "dc cipapa, xzr"); + check(0xd50e7ebf, "dc cigdpapa, xzr"); + } + + #[test] + fn tlbi_named() { + check(0xd5088100, "tlbi vmalle1os, x0"); + check(0xd50881e0, "tlbi vaale1os, x0"); + check(0xd5088300, "tlbi vmalle1is, x0"); + check(0xd50883e0, "tlbi vaale1is, x0"); + check(0xd5088620, "tlbi rvae1, x0"); + check(0xd5088720, "tlbi vae1, x0"); + check(0xd50c8020, "tlbi ipas2e1is, x0"); + check(0xd50c8120, "tlbi vae2os, x0"); + check(0xd50c8240, "tlbi vmallws2e1is, x0"); + check(0xd50c83a0, "tlbi vale2is, x0"); + check(0xd50c8460, "tlbi ripas2e1os, x0"); + check(0xd50c8520, "tlbi rvae2os, x0"); + check(0xd50c86a0, "tlbi rvale2, x0"); + check(0xd50c87c0, "tlbi vmalls12e1"); + check(0xd50e8220, "tlbi rvae3is, x0"); + check(0xd50e8460, "tlbi rpaos, x0"); + check(0xd50e86a0, "tlbi rvale3, x0"); + check(0xd508811f, "tlbi vmalle1os"); + check(0xd50881ff, "tlbi vaale1os, xzr"); + check(0xd508831f, "tlbi vmalle1is"); + check(0xd50883ff, "tlbi vaale1is, xzr"); + check(0xd508863f, "tlbi rvae1, xzr"); + check(0xd508873f, "tlbi vae1, xzr"); + check(0xd50c803f, "tlbi ipas2e1is, xzr"); + check(0xd50c813f, "tlbi vae2os, xzr"); + check(0xd50c825f, "tlbi vmallws2e1is"); + check(0xd50c83bf, "tlbi vale2is, xzr"); + check(0xd50c847f, "tlbi ripas2e1os, xzr"); + check(0xd50c853f, "tlbi rvae2os, xzr"); + check(0xd50c86bf, "tlbi rvale2, xzr"); + check(0xd50c87df, "tlbi vmalls12e1"); + check(0xd50e823f, "tlbi rvae3is, xzr"); + check(0xd50e847f, "tlbi rpaos, xzr"); + check(0xd50e86bf, "tlbi rvale3, xzr"); + } + + #[test] + fn generic_sys() { + check(0xd5087120, "sys #0, C7, C1, #1, x0"); + check(0xd508713f, "sys #0, C7, C1, #1"); + check(0xd50a883f, "sys #2, C8, C8, #1"); + check(0xd50a8840, "sys #2, C8, C8, #2, x0"); + check(0xd50f8fff, "sys #7, C8, C15, #7"); + check(0xd50f8fe0, "sys #7, C8, C15, #7, x0"); + check(0xd50b7420, "dc zva, x0"); + check(0xd50b743f, "dc zva, xzr"); + check(0xd5087800, "at s1e1r, x0"); + check(0xd5087820, "at s1e1w, x0"); + check(0xd5087840, "at s1e0r, x0"); + check(0xd508785f, "at s1e0r, xzr"); + check(0xd5087860, "at s1e0w, x0"); + } + + #[test] + fn exceptions() { + check(0xd4000002, "hvc #0x0"); + check(0xd4000001, "svc #0x0"); + check(0xd4001001, "svc #0x80"); + check(0xd4200020, "brk #0x1"); + check(0xd4210000, "brk #0x800"); + check(0xd43fffe0, "brk #0xffff"); + check(0xd69f03e0, "eret"); + } + + #[test] + fn pointer_authentication() { + check(0xdac10000, "pacia x0, x0"); + check(0xdac10020, "pacia x0, x1"); + check(0xdac103e0, "pacia x0, sp"); + check(0xdac10400, "pacib x0, x0"); + check(0xdac10800, "pacda x0, x0"); + check(0xdac10c00, "pacdb x0, x0"); + check(0xdac11000, "autia x0, x0"); + check(0xdac11400, "autib x0, x0"); + check(0xdac11800, "autda x0, x0"); + check(0xdac11c00, "autdb x0, x0"); + check(0xdac113e0, "autia x0, sp"); + check(0xdac123e0, "paciza x0"); + check(0xdac127e0, "pacizb x0"); + check(0xdac12be0, "pacdza x0"); + check(0xdac12fe0, "pacdzb x0"); + check(0xdac133e0, "autiza x0"); + check(0xdac137e0, "autizb x0"); + check(0xdac13be0, "autdza x0"); + check(0xdac13fe0, "autdzb x0"); + check(0xdac143e0, "xpaci x0"); + check(0x9ac033e0, "pacga x0, xzr, x0"); + check(0x9adf3000, "pacga x0, x0, sp"); + check(0xd71f0800, "braa x0, x0"); + check(0xd71f0fe3, "brab xzr, x3"); + check(0xd65f0bff, "retaa"); + check(0xd65f0fff, "retab"); + check(0xd69f0bff, "eretaa"); + check(0xd69f0fff, "eretab"); + check(0xd503231f, "paciaz"); + check(0xd503233f, "paciasp"); + check(0xd503235f, "pacibz"); + check(0xd503237f, "pacibsp"); + check(0xd503239f, "autiaz"); + check(0xd50323bf, "autiasp"); + check(0xd50323df, "autibz"); + check(0xd50323ff, "autibsp"); + check(0xd503211f, "pacia1716"); + check(0xd503215f, "pacib1716"); + check(0xd503219f, "autia1716"); + check(0xd50321df, "autib1716"); + check(0xd50320ff, "xpaclri"); + // The register forms of the same operations are not the hint spellings. + check(0xdac103fe, "pacia x30, sp"); + check(0xdac10211, "pacia x17, x16"); + check(0xdac10611, "pacib x17, x16"); + check(0xdac113fe, "autia x30, sp"); + check(0xdac11211, "autia x17, x16"); + check(0xdac11611, "autib x17, x16"); + check(0xdac123fe, "paciza x30"); + check(0xdac127fe, "pacizb x30"); + check(0xdac133fe, "autiza x30"); + check(0xdac137fe, "autizb x30"); + check(0xdac143fe, "xpaci x30"); + check(0xdac147e0, "xpacd x0"); + check(0xdac147fe, "xpacd x30"); + } + + #[test] + fn register_branches() { + check(0xd61f0000, "br x0"); + check(0xd61f0220, "br x17"); + check(0xd61f03e0, "br xzr"); + check(0xd63f0000, "blr x0"); + check(0xd63f03c0, "blr x30"); + check(0xd63f03e0, "blr xzr"); + check(0xd65f0000, "ret x0"); + check(0xd65f03c0, "ret"); + check(0xd65f0220, "ret x17"); + check(0xd65f03e0, "ret xzr"); + } + + #[test] + fn debug_and_apple_zero_registers_keep_their_spelling() { + check(0xd5300080, "mrs x0, dbgbvr0_el1"); + check(0xd53000a0, "mrs x0, dbgbcr0_el1"); + check(0xd53000c0, "mrs x0, dbgwvr0_el1"); + check(0xd53000e0, "mrs x0, dbgwcr0_el1"); + check(0xd5300fe0, "mrs x0, dbgwcr15_el1"); + check(0xd5300ac1, "mrs x1, dbgwvr10_el1"); + check(0xd5100180, "msr dbgbvr1_el1, x0"); + check(0xd51001a0, "msr dbgbcr1_el1, x0"); + check(0xd51001c0, "msr dbgwvr1_el1, x0"); + check(0xd51001e0, "msr dbgwcr1_el1, x0"); + check(0xd53df600, "mrs x0, s3_5_c15_c6_0"); + check(0xd53ff680, "mrs x0, s3_7_c15_c6_4"); + check(0xd539fd00, "mrs x0, s3_1_c15_c13_0"); + check(0xd53df120, "mrs x0, s3_5_c15_c1_1"); + } + + #[test] + fn implementation_defined_registers() { + check(0xd539f000, "mrs x0, s3_1_c15_c0_0"); + check(0xd539f600, "mrs x0, s3_1_c15_c6_0"); + check(0xd53afa00, "mrs x0, s3_2_c15_c10_0"); + check(0xd53df320, "mrs x0, s3_5_c15_c3_1"); + check(0xd53ff080, "mrs x0, s3_7_c15_c0_4"); + check(0xd539fd00, "mrs x0, s3_1_c15_c13_0"); + check(0xd53df120, "mrs x0, s3_5_c15_c1_1"); + check(0xd53df300, "mrs x0, s3_5_c15_c3_0"); + check(0xd53df700, "mrs x0, s3_5_c15_c7_0"); + check(0xd53cf3a0, "mrs x0, s3_4_c15_c3_5"); + check(0xd53effe0, "mrs x0, s3_6_c15_c15_7"); + check(0xd538f000, "mrs x0, s3_0_c15_c0_0"); + check(0xd538ffa0, "mrs x0, s3_0_c15_c15_5"); + check(0xd53bfa00, "mrs x0, s3_3_c15_c10_0"); + check(0xd539f840, "mrs x0, s3_1_c15_c8_2"); + check(0xd5382100, "mrs x0, apiakeylo_el1"); + check(0xd5382120, "mrs x0, apiakeyhi_el1"); + check(0xd5382140, "mrs x0, apibkeylo_el1"); + check(0xd5382160, "mrs x0, apibkeyhi_el1"); + check(0xd5382200, "mrs x0, apdakeylo_el1"); + check(0xd5382220, "mrs x0, apdakeyhi_el1"); + check(0xd5382240, "mrs x0, apdbkeylo_el1"); + check(0xd5382260, "mrs x0, apdbkeyhi_el1"); + check(0xd5382300, "mrs x0, apgakeylo_el1"); + check(0xd5382320, "mrs x0, apgakeyhi_el1"); + check(0xd519f000, "msr s3_1_c15_c0_0, x0"); + check(0xd519f600, "msr s3_1_c15_c6_0, x0"); + check(0xd51afa00, "msr s3_2_c15_c10_0, x0"); + check(0xd5182100, "msr apiakeylo_el1, x0"); + check(0xd5182120, "msr apiakeyhi_el1, x0"); + } +} diff --git a/src/isa/aarch64/src/format/vector.rs b/src/isa/aarch64/src/format/vector.rs new file mode 100644 index 0000000..50c4771 --- /dev/null +++ b/src/isa/aarch64/src/format/vector.rs @@ -0,0 +1,1033 @@ +//! Advanced SIMD data processing: modified immediates, moves, `dup`/`ins`/`umov`, +//! `ext`, `tbl`/`tbx`, the integer three-same group, compares against zero, and the +//! integer instruction forms of `crate::simd`. +//! +//! Owns `SimdImm`, `SimdCompareZero`, `SimdDup`, `SimdDupElement`, +//! `SimdInsert`, `SimdInsertElement`, `SimdExtract`, `SimdExt`, `SimdAlu`, `SimdTable` +//! and `Simd`. + +use super::Context; +use super::reg::{fp_scalar, gpr, vector}; +use crate::decode::{ + ImmCombine, ImmForm, Instruction, SimdAlu, SimdAluOp, SimdImmediate, SimdTable, Width, +}; +use crate::simd::{Long, Narrow, Op, Permute, Scalar, Zero}; +use core::fmt::{self, Formatter, Write}; + +pub(super) fn write( + f: &mut Formatter<'_>, + cx: &Context<'_>, + instruction: &Instruction, +) -> fmt::Result { + let _ = cx; + match *instruction { + Instruction::SimdImm(imm) => immediate(f, imm), + Instruction::SimdCompareZero(c) => { + f.write_str("cmeq ")?; + two_same(f, c.rd, c.rn, c.size, c.q)?; + f.write_str(", #0") + } + Instruction::SimdDup { rd, rn, esize, q } => { + f.write_str("dup ")?; + vector(f, rd, esize & 3, q)?; + f.write_str(", ")?; + gpr(f, lane_width(esize), rn) + } + Instruction::SimdDupElement(d) => { + f.write_str("dup ")?; + vector(f, d.rd, d.esize & 3, d.q)?; + f.write_str(", ")?; + element(f, d.rn, d.esize, d.index) + } + Instruction::SimdInsert(i) => { + f.write_str("mov ")?; + element(f, i.rd, i.esize, i.index)?; + f.write_str(", ")?; + gpr(f, lane_width(i.esize), i.rn) + } + Instruction::SimdInsertElement(i) => { + f.write_str("mov ")?; + element(f, i.rd, i.esize, i.dst)?; + f.write_str(", ")?; + element(f, i.rn, i.esize, i.src) + } + // `umov` of a word or doubleword lane is spelled `mov`. + Instruction::SimdExtract(e) => { + f.write_str(if e.esize & 3 >= 2 { "mov " } else { "umov " })?; + gpr(f, lane_width(e.esize), e.rd)?; + f.write_str(", ")?; + element(f, e.rn, e.esize, e.index) + } + Instruction::SimdExt(e) => { + f.write_str("ext ")?; + vector(f, e.rd, 0, e.q)?; + f.write_str(", ")?; + vector(f, e.rn, 0, e.q)?; + f.write_str(", ")?; + vector(f, e.rm, 0, e.q)?; + write!(f, ", #{}", e.imm) + } + Instruction::SimdAlu(alu) => simd_alu(f, alu), + Instruction::SimdTable(t) => table(f, t), + Instruction::Simd(op) => simd(f, op), + _ => unreachable!("{instruction:?} is not an Advanced SIMD vector instruction"), + } +} + +/// `X` for a doubleword lane, `W` for every narrower one. +fn lane_width(esize: u8) -> Width { + if esize & 3 == 3 { + Width::X64 + } else { + Width::W32 + } +} + +/// The element letter of `esize` (log2 of the byte width). +fn lane_letter(esize: u8) -> char { + ['b', 'h', 's', 'd'][usize::from(esize & 3)] +} + +/// One lane of a vector register: `v1.s[2]`. +fn element(f: &mut Formatter<'_>, r: u8, esize: u8, index: u8) -> fmt::Result { + write!(f, "v{r}.{}[{index}]", lane_letter(esize)) +} + +/// `vd.T, vn.T` with one arrangement. +fn two_same(f: &mut Formatter<'_>, rd: u8, rn: u8, size: u8, q: bool) -> fmt::Result { + vector(f, rd, size & 3, q)?; + f.write_str(", ")?; + vector(f, rn, size & 3, q) +} + +/// `vd.T, vn.T, vm.T` with one arrangement. +fn three_same(f: &mut Formatter<'_>, rd: u8, rn: u8, rm: u8, size: u8, q: bool) -> fmt::Result { + two_same(f, rd, rn, size, q)?; + f.write_str(", ")?; + vector(f, rm, size & 3, q) +} + +fn simd_alu(f: &mut Formatter<'_>, alu: SimdAlu) -> fmt::Result { + use SimdAluOp::*; + let SimdAlu { + op, + rd, + rn, + rm, + size, + q, + } = alu; + let size = size & 3; + let name = match op { + Add => "add", + Sub => "sub", + Mul => "mul", + Mla => "mla", + Mls => "mls", + And => "and", + Bic => "bic", + Orr if rn == rm => { + f.write_str("mov ")?; + return two_same(f, rd, rn, 0, q); + } + Orr => "orr", + Orn => "orn", + Eor => "eor", + Bsl => "bsl", + Bit => "bit", + Bif => "bif", + CmEq => "cmeq", + CmGt => "cmgt", + CmGe => "cmge", + CmHi => "cmhi", + CmHs => "cmhs", + CmTst => "cmtst", + UMin => "umin", + UMax => "umax", + SMin => "smin", + SMax => "smax", + UAbd => "uabd", + SAbd => "sabd", + SQAdd => "sqadd", + UQAdd => "uqadd", + SQSub => "sqsub", + UQSub => "uqsub", + SHAdd => "shadd", + UHAdd => "uhadd", + SHSub => "shsub", + UHSub => "uhsub", + SRHAdd => "srhadd", + URHAdd => "urhadd", + SSHl => "sshl", + USHl => "ushl", + SQShl => "sqshl", + UQShl => "uqshl", + SQRShl => "sqrshl", + UQRShl => "uqrshl", + AddV => "addv", + UMinV => "uminv", + UMaxV => "umaxv", + UAddLV => "uaddlv", + UMaxP => "umaxp", + SMaxP => "smaxp", + UMinP => "uminp", + SMinP => "sminp", + AddP => "addp", + }; + write!(f, "{name} ")?; + match op { + // Across lanes: a scalar result, as wide as the lanes (twice as wide for + // `uaddlv`), from a vector. + AddV | UMinV | UMaxV | UAddLV => { + let bytes = 1u8 << (size + u8::from(op == UAddLV)); + fp_scalar(f, bytes, rd)?; + f.write_str(", ")?; + vector(f, rn, size, q) + } + And | Bic | Orr | Orn | Eor | Bsl | Bit | Bif => three_same(f, rd, rn, rm, 0, q), + _ => three_same(f, rd, rn, rm, size, q), + } +} + +fn table(f: &mut Formatter<'_>, t: SimdTable) -> fmt::Result { + f.write_str(if t.extend { "tbx " } else { "tbl " })?; + vector(f, t.rd, 0, t.q)?; + f.write_str(", {")?; + vector(f, t.rn, 0, true)?; + if t.len & 3 != 0 { + f.write_str("-")?; + vector(f, t.rn.wrapping_add(t.len & 3) & 31, 0, true)?; + } + f.write_str("}, ")?; + vector(f, t.rm, 0, t.q) +} + +/// The mnemonic suffix of the upper-half (`2`) form of a long or narrow operation. +fn two(upper: bool) -> &'static str { + if upper { "2" } else { "" } +} + +fn simd(f: &mut Formatter<'_>, op: Op) -> fmt::Result { + match op { + Op::Permute { + kind, + size, + q, + rd, + rn, + rm, + } => { + f.write_str(match kind { + Permute::Uzp1 => "uzp1 ", + Permute::Uzp2 => "uzp2 ", + Permute::Zip1 => "zip1 ", + Permute::Zip2 => "zip2 ", + Permute::Trn1 => "trn1 ", + Permute::Trn2 => "trn2 ", + })?; + three_same(f, rd, rn, rm, size as u8, q) + } + Op::Reverse { + group, + size, + q, + rd, + rn, + } => { + write!(f, "rev{group} ")?; + two_same(f, rd, rn, size as u8, q) + } + Op::Count { q, rd, rn } => { + f.write_str("cnt ")?; + two_same(f, rd, rn, 0, q) + } + Op::Not { q, rd, rn } => { + f.write_str("mvn ")?; + two_same(f, rd, rn, 0, q) + } + Op::ReverseBits { q, rd, rn } => { + f.write_str("rbit ")?; + two_same(f, rd, rn, 0, q) + } + Op::CompareZero { + kind, + size, + q, + rd, + rn, + } => { + f.write_str(match kind { + Zero::Gt => "cmgt ", + Zero::Ge => "cmge ", + Zero::Le => "cmle ", + Zero::Lt => "cmlt ", + })?; + two_same(f, rd, rn, size as u8, q)?; + f.write_str(", #0") + } + Op::Abs { + negate, + size, + q, + rd, + rn, + } => { + f.write_str(if negate { "neg " } else { "abs " })?; + two_same(f, rd, rn, size as u8, q) + } + Op::Narrow { + kind, + size, + upper, + rd, + rn, + } => { + let name = match kind { + Narrow::Plain => "xtn", + Narrow::SignedSat => "sqxtn", + Narrow::UnsignedSat => "uqxtn", + Narrow::SignedToUnsignedSat => "sqxtun", + }; + let size = size as u8 & 3; + write!(f, "{name}{} ", two(upper))?; + vector(f, rd, size, upper)?; + f.write_str(", ")?; + vector(f, rn, size.wrapping_add(1) & 3, true) + } + Op::PairLong { + signed, + accumulate, + size, + q, + rd, + rn, + } => { + let size = size as u8 & 3; + write!( + f, + "{}{} ", + if signed { 's' } else { 'u' }, + if accumulate { "adalp" } else { "addlp" } + )?; + vector(f, rd, size.wrapping_add(1) & 3, q)?; + f.write_str(", ")?; + vector(f, rn, size, q) + } + Op::Long { + kind, + signed, + size, + upper, + rd, + rn, + rm, + } => { + let name = match kind { + Long::Add => "addl", + Long::AddWide => "addw", + Long::Sub => "subl", + Long::SubWide => "subw", + Long::Abd => "abdl", + Long::Mul => "mull", + Long::Mla => "mlal", + Long::Mls => "mlsl", + }; + let size = size as u8 & 3; + write!(f, "{}{name}{} ", if signed { 's' } else { 'u' }, two(upper))?; + vector(f, rd, size.wrapping_add(1) & 3, true)?; + f.write_str(", ")?; + if matches!(kind, Long::AddWide | Long::SubWide) { + vector(f, rn, size.wrapping_add(1) & 3, true)?; + } else { + vector(f, rn, size, upper)?; + } + f.write_str(", ")?; + vector(f, rm, size, upper) + } + Op::ScalarAddp { rd, rn } => write!(f, "addp d{rd}, v{rn}.2d"), + Op::ScalarDup { + size, + index, + rd, + rn, + } => { + let size = size as u8 & 3; + f.write_str("mov ")?; + fp_scalar(f, 1 << size, rd)?; + f.write_str(", ")?; + element(f, rn, size, index as u8) + } + Op::ScalarThree { kind, rd, rn, rm } => { + let name = match kind { + Scalar::Add => "add", + Scalar::Sub => "sub", + Scalar::Gt => "cmgt", + Scalar::Hi => "cmhi", + Scalar::Ge => "cmge", + Scalar::Hs => "cmhs", + Scalar::Eq => "cmeq", + Scalar::Tst => "cmtst", + }; + write!(f, "{name} d{rd}, d{rn}, d{rm}") + } + Op::ShiftImm { + right, + signed, + size, + q, + shift, + rd, + rn, + } => { + f.write_str(match (right, signed) { + (false, _) => "shl ", + (true, true) => "sshr ", + (true, false) => "ushr ", + })?; + two_same(f, rd, rn, size as u8, q)?; + write!(f, ", #{shift}") + } + Op::ShiftLong { + signed, + size, + shift, + upper, + rd, + rn, + } => { + let size = size as u8 & 3; + let sign = if signed { 's' } else { 'u' }; + if shift == 0 { + write!(f, "{sign}xtl{} ", two(upper))?; + } else { + write!(f, "{sign}shll{} ", two(upper))?; + } + vector(f, rd, size.wrapping_add(1) & 3, true)?; + f.write_str(", ")?; + vector(f, rn, size, upper)?; + if shift != 0 { + write!(f, ", #{shift}")?; + } + Ok(()) + } + Op::ShiftNarrow { + size, + shift, + upper, + rd, + rn, + } => { + let size = size as u8 & 3; + write!(f, "shrn{} ", two(upper))?; + vector(f, rd, size, upper)?; + f.write_str(", ")?; + vector(f, rn, size.wrapping_add(1) & 3, true)?; + write!(f, ", #{shift}") + } + } +} + +/// A floating-point immediate the way binutils prints it: C's `%.18e`. +fn float_immediate(f: &mut Formatter<'_>, value: f64) -> fmt::Result { + // `fmov` immediates are always finite; other bit patterns only come from + // hand-built values and print as C's `%f` would. + if !value.is_finite() { + return f.write_str(if value.is_nan() { + "nan" + } else if value < 0.0 { + "-inf" + } else { + "inf" + }); + } + let mut text = Text { + bytes: [0; 64], + len: 0, + }; + write!(text, "{value:.18e}")?; + let text = core::str::from_utf8(&text.bytes[..text.len]).map_err(|_| fmt::Error)?; + let (mantissa, exponent) = text.split_once('e').ok_or(fmt::Error)?; + let exponent: i32 = exponent.parse().map_err(|_| fmt::Error)?; + let sign = if exponent < 0 { '-' } else { '+' }; + write!(f, "{mantissa}e{sign}{:02}", exponent.unsigned_abs()) +} + +/// A fixed buffer to format a float into; every value printed here fits. +struct Text { + bytes: [u8; 64], + len: usize, +} + +impl Write for Text { + fn write_str(&mut self, s: &str) -> fmt::Result { + let end = self.len.checked_add(s.len()).ok_or(fmt::Error)?; + self.bytes + .get_mut(self.len..end) + .ok_or(fmt::Error)? + .copy_from_slice(s.as_bytes()); + self.len = end; + Ok(()) + } +} + +/// `name v., #[, #]`. +fn lane_form( + f: &mut Formatter<'_>, + name: &str, + rd: u8, + lanes: &str, + imm8: u8, + kind: &str, + shift: u8, +) -> fmt::Result { + write!(f, "{name} v{rd}.{lanes}, #{imm8:#x}")?; + if shift != 0 { + write!(f, ", {kind} #{shift}")?; + } + Ok(()) +} + +/// A half-precision value (as `fmov`'s expanded immediates are: always normal) as +/// a double. +fn half_to_f64(bits: u16) -> f64 { + let sign = if bits & 0x8000 != 0 { -1.0 } else { 1.0 }; + let exponent = u64::from((bits >> 10) & 0x1f); + let fraction = f64::from(bits & 0x3ff); + // 2^(e - 25) and 2^-24 as double exponents. + let scale = |e: u64| f64::from_bits((1023 + e - 25) << 52); + match exponent { + 0 => sign * fraction * scale(1), + 31 if fraction == 0.0 => sign * f64::INFINITY, + 31 => f64::NAN, + e => sign * (1024.0 + fraction) * scale(e), + } +} + +/// A vector modified immediate, spelled from the encoding the decoder kept +/// (`form`, `imm8`, `combine`, `q`); `low` is only read for the 64-bit `movi` +/// and the `fmov` forms, whose operand is the expanded pattern. +fn immediate(f: &mut Formatter<'_>, imm: SimdImmediate) -> fmt::Result { + let SimdImmediate { + rd, + imm8, + form, + combine, + q, + low, + .. + } = imm; + let name = match combine { + ImmCombine::Set => "movi", + ImmCombine::SetInverted => "mvni", + ImmCombine::Or => "orr", + ImmCombine::AndNot => "bic", + }; + match form { + ImmForm::Byte => lane_form(f, name, rd, if q { "16b" } else { "8b" }, imm8, "lsl", 0), + ImmForm::Half { shift } => { + lane_form(f, name, rd, if q { "8h" } else { "4h" }, imm8, "lsl", shift) + } + ImmForm::Word { shift } => { + lane_form(f, name, rd, if q { "4s" } else { "2s" }, imm8, "lsl", shift) + } + ImmForm::WordMask { shift } => { + lane_form(f, name, rd, if q { "4s" } else { "2s" }, imm8, "msl", shift) + } + // The 64-bit lane: the whole expanded constant, as a scalar when `Q` is clear. + ImmForm::Double if q => write!(f, "{name} v{rd}.2d, #{low:#x}"), + ImmForm::Double => write!(f, "{name} d{rd}, #{low:#x}"), + ImmForm::FmovHalf => { + write!(f, "fmov v{rd}.{}, #", if q { "8h" } else { "4h" })?; + float_immediate(f, half_to_f64(low as u16)) + } + ImmForm::FmovSingle => { + write!(f, "fmov v{rd}.{}, #", if q { "4s" } else { "2s" })?; + float_immediate(f, f64::from(f32::from_bits(low as u32))) + } + ImmForm::FmovDouble => { + write!(f, "fmov v{rd}.2d, #")?; + float_immediate(f, f64::from_bits(low)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::decode::{ImmCombine, SimdAlu, SimdDupElement, SimdExtract, SimdInsert, decode}; + + struct Show<'a>(&'a Instruction); + + impl core::fmt::Display for Show<'_> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write( + f, + &Context { + pc: None, + symbols: &(), + }, + self.0, + ) + } + } + + /// The rendered text, kept in a fixed buffer. + struct Rendered(Text); + + impl Rendered { + fn as_str(&self) -> &str { + core::str::from_utf8(&self.0.bytes[..self.0.len]).unwrap() + } + } + + impl PartialEq<&str> for Rendered { + fn eq(&self, other: &&str) -> bool { + self.as_str() == *other + } + } + + impl core::fmt::Debug for Rendered { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.as_str()) + } + } + + fn text(instruction: &Instruction) -> Rendered { + let mut text = Text { + bytes: [0; 64], + len: 0, + }; + write!(text, "{}", Show(instruction)).unwrap(); + Rendered(text) + } + + /// Words and the `objdump` text (binutils 2.46) for them. Excluded on purpose: + /// `movi v0.16b, #0xff`, `movi d31, #0xffffffffffffffff` and the other + /// encodings that expand to the same pattern as a form listed here (`mvni v0.4s, + /// #0`); the decoder keeps only the expanded pattern, so it cannot tell them apart. + #[test] + fn matches_objdump() { + let cases: [(u32, &str); 247] = [ + (0x4f000400, "movi v0.4s, #0x0"), + (0x0f000401, "movi v1.2s, #0x0"), + (0x4f05657f, "movi v31.4s, #0xab, lsl #24"), + (0x0f002420, "movi v0.2s, #0x1, lsl #8"), + (0x4f0347e0, "movi v0.4s, #0x7f, lsl #16"), + (0x6f0747e0, "mvni v0.4s, #0xff, lsl #16"), + (0x2f002423, "mvni v3.2s, #0x1, lsl #8"), + (0x6f000423, "mvni v3.4s, #0x1"), + (0x6f000400, "mvni v0.4s, #0x0"), + (0x2f000400, "mvni v0.2s, #0x0"), + (0x4f008640, "movi v0.8h, #0x12"), + (0x0f00a420, "movi v0.4h, #0x1, lsl #8"), + (0x6f00a460, "mvni v0.8h, #0x3, lsl #8"), + (0x2f008460, "mvni v0.4h, #0x3"), + (0x4f00c420, "movi v0.4s, #0x1, msl #8"), + (0x0f07d7e0, "movi v0.2s, #0xff, msl #16"), + (0x6f00d420, "mvni v0.4s, #0x1, msl #16"), + (0x2f00c4e0, "mvni v0.2s, #0x7, msl #8"), + (0x4f00e640, "movi v0.16b, #0x12"), + (0x0f00e640, "movi v0.8b, #0x12"), + (0x2f05e4a0, "movi d0, #0xff00ff0000ff00ff"), + (0x6f05e4a0, "movi v0.2d, #0xff00ff0000ff00ff"), + (0x6f05e600, "movi v0.2d, #0xff00ffff00000000"), + (0x4f03f600, "fmov v0.4s, #1.000000000000000000e+00"), + (0x0f04f400, "fmov v0.2s, #-2.000000000000000000e+00"), + (0x6f01f7e0, "fmov v0.2d, #3.100000000000000000e+01"), + (0x6f06f400, "fmov v0.2d, #-1.250000000000000000e-01"), + (0x4f03f401, "fmov v1.4s, #5.000000000000000000e-01"), + (0x4f03f7e1, "fmov v1.4s, #1.937500000000000000e+00"), + (0x6f07f7e1, "fmov v1.2d, #-1.937500000000000000e+00"), + (0x0f02f401, "fmov v1.2s, #1.250000000000000000e-01"), + (0x4f001420, "orr v0.4s, #0x1"), + (0x4f0777e0, "orr v0.4s, #0xff, lsl #24"), + (0x0f0054e0, "orr v0.2s, #0x7, lsl #16"), + (0x4f00b420, "orr v0.8h, #0x1, lsl #8"), + (0x0f049400, "orr v0.4h, #0x80"), + (0x2f005440, "bic v0.2s, #0x2, lsl #16"), + (0x2f0094e0, "bic v0.4h, #0x7"), + (0x6f00361f, "bic v31.4s, #0x10, lsl #8"), + (0x4e209801, "cmeq v1.16b, v0.16b, #0"), + (0x0e209801, "cmeq v1.8b, v0.8b, #0"), + (0x0ea09be1, "cmeq v1.2s, v31.2s, #0"), + (0x4ee0981f, "cmeq v31.2d, v0.2d, #0"), + (0x4e609801, "cmeq v1.8h, v0.8h, #0"), + (0x4e040c20, "dup v0.4s, w1"), + (0x4e080c20, "dup v0.2d, x1"), + (0x0e010fe0, "dup v0.8b, wzr"), + (0x4e010c60, "dup v0.16b, w3"), + (0x0e020c40, "dup v0.4h, w2"), + (0x0e040fff, "dup v31.2s, wzr"), + (0x4e140420, "dup v0.4s, v1.s[2]"), + (0x4e1f0420, "dup v0.16b, v1.b[15]"), + (0x4e1807e0, "dup v0.2d, v31.d[1]"), + (0x4e1e0420, "dup v0.8h, v1.h[7]"), + (0x0e040420, "dup v0.2s, v1.s[0]"), + (0x0e0e0420, "dup v0.4h, v1.h[3]"), + (0x4e0c1c20, "mov v0.s[1], w1"), + (0x4e181fe0, "mov v0.d[1], xzr"), + (0x4e1f1c40, "mov v0.b[15], w2"), + (0x4e021fe0, "mov v0.h[0], wzr"), + (0x4e081fdf, "mov v31.d[0], x30"), + (0x6e0c4420, "mov v0.s[1], v1.s[2]"), + (0x6e1f07e0, "mov v0.b[15], v31.b[0]"), + (0x6e180420, "mov v0.d[1], v1.d[0]"), + (0x6e1e3420, "mov v0.h[7], v1.h[3]"), + (0x6e1c6420, "mov v0.s[3], v1.s[3]"), + (0x0e073c20, "umov w0, v1.b[3]"), + (0x0e1e3c20, "umov w0, v1.h[7]"), + (0x0e1c3c20, "mov w0, v1.s[3]"), + (0x4e183c20, "mov x0, v1.d[1]"), + (0x0e043ffe, "mov w30, v31.s[0]"), + (0x0e1f3fe0, "umov w0, v31.b[15]"), + (0x6e021820, "ext v0.16b, v1.16b, v2.16b, #3"), + (0x6e027820, "ext v0.16b, v1.16b, v2.16b, #15"), + (0x6e1f03ff, "ext v31.16b, v31.16b, v31.16b, #0"), + (0x4e228420, "add v0.16b, v1.16b, v2.16b"), + (0x4ee28420, "add v0.2d, v1.2d, v2.2d"), + (0x0e628420, "add v0.4h, v1.4h, v2.4h"), + (0x6e628420, "sub v0.8h, v1.8h, v2.8h"), + (0x4ea29c20, "mul v0.4s, v1.4s, v2.4s"), + (0x0ea29420, "mla v0.2s, v1.2s, v2.2s"), + (0x6e229420, "mls v0.16b, v1.16b, v2.16b"), + (0x4e221c20, "and v0.16b, v1.16b, v2.16b"), + (0x0e221c20, "and v0.8b, v1.8b, v2.8b"), + (0x4e621c20, "bic v0.16b, v1.16b, v2.16b"), + (0x4ea21c20, "orr v0.16b, v1.16b, v2.16b"), + (0x0ea21c20, "orr v0.8b, v1.8b, v2.8b"), + (0x4ea11c20, "mov v0.16b, v1.16b"), + (0x0ebf1fff, "mov v31.8b, v31.8b"), + (0x4ee21c20, "orn v0.16b, v1.16b, v2.16b"), + (0x6e221c20, "eor v0.16b, v1.16b, v2.16b"), + (0x6e621c20, "bsl v0.16b, v1.16b, v2.16b"), + (0x2ea21c20, "bit v0.8b, v1.8b, v2.8b"), + (0x6ee21c20, "bif v0.16b, v1.16b, v2.16b"), + (0x6ea28c20, "cmeq v0.4s, v1.4s, v2.4s"), + (0x4ee23420, "cmgt v0.2d, v1.2d, v2.2d"), + (0x4e623c20, "cmge v0.8h, v1.8h, v2.8h"), + (0x6e223420, "cmhi v0.16b, v1.16b, v2.16b"), + (0x6ea23c20, "cmhs v0.4s, v1.4s, v2.4s"), + (0x0ea28c20, "cmtst v0.2s, v1.2s, v2.2s"), + (0x6e226c20, "umin v0.16b, v1.16b, v2.16b"), + (0x6e626420, "umax v0.8h, v1.8h, v2.8h"), + (0x4ea26c20, "smin v0.4s, v1.4s, v2.4s"), + (0x0e226420, "smax v0.8b, v1.8b, v2.8b"), + (0x6e227420, "uabd v0.16b, v1.16b, v2.16b"), + (0x0e627420, "sabd v0.4h, v1.4h, v2.4h"), + (0x4ee20c20, "sqadd v0.2d, v1.2d, v2.2d"), + (0x6e220c20, "uqadd v0.16b, v1.16b, v2.16b"), + (0x4e622c20, "sqsub v0.8h, v1.8h, v2.8h"), + (0x6ea22c20, "uqsub v0.4s, v1.4s, v2.4s"), + (0x4e220420, "shadd v0.16b, v1.16b, v2.16b"), + (0x6e620420, "uhadd v0.8h, v1.8h, v2.8h"), + (0x4ea22420, "shsub v0.4s, v1.4s, v2.4s"), + (0x2e222420, "uhsub v0.8b, v1.8b, v2.8b"), + (0x4e221420, "srhadd v0.16b, v1.16b, v2.16b"), + (0x6e621420, "urhadd v0.8h, v1.8h, v2.8h"), + (0x4ee24420, "sshl v0.2d, v1.2d, v2.2d"), + (0x6e224420, "ushl v0.16b, v1.16b, v2.16b"), + (0x4ea24c20, "sqshl v0.4s, v1.4s, v2.4s"), + (0x6e624c20, "uqshl v0.8h, v1.8h, v2.8h"), + (0x4e225c20, "sqrshl v0.16b, v1.16b, v2.16b"), + (0x6ee25c20, "uqrshl v0.2d, v1.2d, v2.2d"), + (0x4ea2bc20, "addp v0.4s, v1.4s, v2.4s"), + (0x4e22bc20, "addp v0.16b, v1.16b, v2.16b"), + (0x6e62a420, "umaxp v0.8h, v1.8h, v2.8h"), + (0x4ea2a420, "smaxp v0.4s, v1.4s, v2.4s"), + (0x6e22ac20, "uminp v0.16b, v1.16b, v2.16b"), + (0x0ea2ac20, "sminp v0.2s, v1.2s, v2.2s"), + (0x4e31b820, "addv b0, v1.16b"), + (0x0e71b83f, "addv h31, v1.4h"), + (0x4eb1bbe0, "addv s0, v31.4s"), + (0x0e31b820, "addv b0, v1.8b"), + (0x6e71a820, "uminv h0, v1.8h"), + (0x6e30a820, "umaxv b0, v1.16b"), + (0x6eb0a820, "umaxv s0, v1.4s"), + (0x6e303820, "uaddlv h0, v1.16b"), + (0x6e703820, "uaddlv s0, v1.8h"), + (0x6eb03820, "uaddlv d0, v1.4s"), + (0x2e303820, "uaddlv h0, v1.8b"), + (0x4e030020, "tbl v0.16b, {v1.16b}, v3.16b"), + (0x4e032020, "tbl v0.16b, {v1.16b-v2.16b}, v3.16b"), + (0x4e044020, "tbl v0.16b, {v1.16b-v3.16b}, v4.16b"), + (0x0e1f6020, "tbl v0.8b, {v1.16b-v4.16b}, v31.8b"), + (0x4e0323e0, "tbl v0.16b, {v31.16b-v0.16b}, v3.16b"), + (0x4e0373c0, "tbx v0.16b, {v30.16b-v1.16b}, v3.16b"), + (0x0e0053bf, "tbx v31.8b, {v29.16b-v31.16b}, v0.8b"), + (0x0e031020, "tbx v0.8b, {v1.16b}, v3.8b"), + (0x4e021820, "uzp1 v0.16b, v1.16b, v2.16b"), + (0x4e425820, "uzp2 v0.8h, v1.8h, v2.8h"), + (0x4e823820, "zip1 v0.4s, v1.4s, v2.4s"), + (0x4ec27820, "zip2 v0.2d, v1.2d, v2.2d"), + (0x0e022820, "trn1 v0.8b, v1.8b, v2.8b"), + (0x0e826820, "trn2 v0.2s, v1.2s, v2.2s"), + (0x4e201820, "rev16 v0.16b, v1.16b"), + (0x6e600820, "rev32 v0.8h, v1.8h"), + (0x2e200820, "rev32 v0.8b, v1.8b"), + (0x4ea00820, "rev64 v0.4s, v1.4s"), + (0x4e200820, "rev64 v0.16b, v1.16b"), + (0x4e205820, "cnt v0.16b, v1.16b"), + (0x0e205820, "cnt v0.8b, v1.8b"), + (0x6e205820, "mvn v0.16b, v1.16b"), + (0x2e205820, "mvn v0.8b, v1.8b"), + (0x6e605820, "rbit v0.16b, v1.16b"), + (0x2e605820, "rbit v0.8b, v1.8b"), + (0x4ea08820, "cmgt v0.4s, v1.4s, #0"), + (0x6e208820, "cmge v0.16b, v1.16b, #0"), + (0x6e609820, "cmle v0.8h, v1.8h, #0"), + (0x4ee0a820, "cmlt v0.2d, v1.2d, #0"), + (0x0ea0a820, "cmlt v0.2s, v1.2s, #0"), + (0x4ea0b820, "abs v0.4s, v1.4s"), + (0x4ee0b820, "abs v0.2d, v1.2d"), + (0x6e20b820, "neg v0.16b, v1.16b"), + (0x2e60b820, "neg v0.4h, v1.4h"), + (0x0e212820, "xtn v0.8b, v1.8h"), + (0x4e212820, "xtn2 v0.16b, v1.8h"), + (0x0e612820, "xtn v0.4h, v1.4s"), + (0x4ea12820, "xtn2 v0.4s, v1.2d"), + (0x0ea14820, "sqxtn v0.2s, v1.2d"), + (0x4e614820, "sqxtn2 v0.8h, v1.4s"), + (0x2e214820, "uqxtn v0.8b, v1.8h"), + (0x6ea14820, "uqxtn2 v0.4s, v1.2d"), + (0x2e612820, "sqxtun v0.4h, v1.4s"), + (0x6e212820, "sqxtun2 v0.16b, v1.8h"), + (0x4e202820, "saddlp v0.8h, v1.16b"), + (0x0e202820, "saddlp v0.4h, v1.8b"), + (0x6e602820, "uaddlp v0.4s, v1.8h"), + (0x2ea02820, "uaddlp v0.1d, v1.2s"), + (0x6ea02820, "uaddlp v0.2d, v1.4s"), + (0x4e606820, "sadalp v0.4s, v1.8h"), + (0x6e206820, "uadalp v0.8h, v1.16b"), + (0x0e220020, "saddl v0.8h, v1.8b, v2.8b"), + (0x4e220020, "saddl2 v0.8h, v1.16b, v2.16b"), + (0x2e620020, "uaddl v0.4s, v1.4h, v2.4h"), + (0x6ea20020, "uaddl2 v0.2d, v1.4s, v2.4s"), + (0x0ea22020, "ssubl v0.2d, v1.2s, v2.2s"), + (0x6e622020, "usubl2 v0.4s, v1.8h, v2.8h"), + (0x0e221020, "saddw v0.8h, v1.8h, v2.8b"), + (0x4ea21020, "saddw2 v0.2d, v1.2d, v2.4s"), + (0x2e621020, "uaddw v0.4s, v1.4s, v2.4h"), + (0x6e221020, "uaddw2 v0.8h, v1.8h, v2.16b"), + (0x0ea23020, "ssubw v0.2d, v1.2d, v2.2s"), + (0x6e623020, "usubw2 v0.4s, v1.4s, v2.8h"), + (0x0e227020, "sabdl v0.8h, v1.8b, v2.8b"), + (0x6e627020, "uabdl2 v0.4s, v1.8h, v2.8h"), + (0x0e628020, "smlal v0.4s, v1.4h, v2.4h"), + (0x6ea28020, "umlal2 v0.2d, v1.4s, v2.4s"), + (0x0e22a020, "smlsl v0.8h, v1.8b, v2.8b"), + (0x6e62a020, "umlsl2 v0.4s, v1.8h, v2.8h"), + (0x0ea2c020, "smull v0.2d, v1.2s, v2.2s"), + (0x6e22c020, "umull2 v0.8h, v1.16b, v2.16b"), + (0x5ef1b820, "addp d0, v1.2d"), + (0x5e070420, "mov b0, v1.b[3]"), + (0x5e1e043f, "mov h31, v1.h[7]"), + (0x5e140420, "mov s0, v1.s[2]"), + (0x5e1807e0, "mov d0, v31.d[1]"), + (0x5ee28420, "add d0, d1, d2"), + (0x7ee28420, "sub d0, d1, d2"), + (0x5ee23420, "cmgt d0, d1, d2"), + (0x7ee23420, "cmhi d0, d1, d2"), + (0x5ee23c20, "cmge d0, d1, d2"), + (0x7ee23c20, "cmhs d0, d1, d2"), + (0x7ee28c20, "cmeq d0, d1, d2"), + (0x5ee28c3f, "cmtst d31, d1, d2"), + (0x4f205420, "shl v0.4s, v1.4s, #0"), + (0x4f0f5420, "shl v0.16b, v1.16b, #7"), + (0x4f7f5420, "shl v0.2d, v1.2d, #63"), + (0x0f1f5420, "shl v0.4h, v1.4h, #15"), + (0x0f080420, "sshr v0.8b, v1.8b, #8"), + (0x4f400420, "sshr v0.2d, v1.2d, #64"), + (0x6f3f0420, "ushr v0.4s, v1.4s, #1"), + (0x6f100420, "ushr v0.8h, v1.8h, #16"), + (0x0f0fa420, "sshll v0.8h, v1.8b, #7"), + (0x4f1fa420, "sshll2 v0.4s, v1.8h, #15"), + (0x0f3fa420, "sshll v0.2d, v1.2s, #31"), + (0x2f09a420, "ushll v0.8h, v1.8b, #1"), + (0x6f21a420, "ushll2 v0.2d, v1.4s, #1"), + (0x0f08a420, "sxtl v0.8h, v1.8b"), + (0x4f10a420, "sxtl2 v0.4s, v1.8h"), + (0x0f20a420, "sxtl v0.2d, v1.2s"), + (0x2f08a420, "uxtl v0.8h, v1.8b"), + (0x6f20a420, "uxtl2 v0.2d, v1.4s"), + (0x2f10a420, "uxtl v0.4s, v1.4h"), + (0x0f088420, "shrn v0.8b, v1.8h, #8"), + (0x4f0f8420, "shrn2 v0.16b, v1.8h, #1"), + (0x0f208420, "shrn v0.2s, v1.2d, #32"), + (0x4f3b8420, "shrn2 v0.4s, v1.2d, #5"), + (0x0f108420, "shrn v0.4h, v1.4s, #16"), + ]; + for (word, expected) in cases { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:08x}: {e:?}")); + assert_eq!(text(&instruction), expected, "{word:08x}"); + } + } + + /// Words verified against GNU `objdump`. + #[test] + fn vector_moves_match_objdump() { + for (word, expected) in [ + (0x2efe1fe0, "bif v0.8b, v31.8b, v30.8b"), + (0x0ee01c20, "orn v0.8b, v1.8b, v0.8b"), + (0x0ea21c20, "orr v0.8b, v1.8b, v2.8b"), + (0x0e211c20, "and v0.8b, v1.8b, v1.8b"), + (0x4e1c1c20, "mov v0.s[3], w1"), + (0x4e083c20, "mov x0, v1.d[0]"), + (0x0e013c20, "umov w0, v1.b[0]"), + ] { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:08x}: {e:?}")); + assert_eq!(text(&instruction), expected, "{word:08x}"); + } + } + + /// Words checked against GNU `objdump`: the modified-immediate forms keep the + /// arrangement and shift, so encodings that expand to the same pattern differ. + #[test] + fn modified_immediates_keep_their_encoding() { + let cases: [(u32, &str); 27] = [ + (0x4f000400, "movi v0.4s, #0x0"), + (0x0f000400, "movi v0.2s, #0x0"), + (0x6f00e401, "movi v1.2d, #0x0"), + (0x2f00e400, "movi d0, #0x0"), + (0x4f00e400, "movi v0.16b, #0x0"), + (0x0f00e460, "movi v0.8b, #0x3"), + (0x6f0707ff, "mvni v31.4s, #0xff"), + (0x6f00045f, "mvni v31.4s, #0x2"), + (0x6f00c400, "mvni v0.4s, #0x0, msl #8"), + (0x4f00d420, "movi v0.4s, #0x1, msl #16"), + (0x0f008420, "movi v0.4h, #0x1"), + (0x4f00a420, "movi v0.8h, #0x1, lsl #8"), + (0x6f00a420, "mvni v0.8h, #0x1, lsl #8"), + (0x4f009420, "orr v0.8h, #0x1"), + (0x4f00b420, "orr v0.8h, #0x1, lsl #8"), + (0x2f00b420, "bic v0.4h, #0x1, lsl #8"), + (0x0f00540a, "orr v10.2s, #0x0, lsl #16"), + (0x0f006400, "movi v0.2s, #0x0, lsl #24"), + (0x4f00f400, "fmov v0.4s, #2.000000000000000000e+00"), + (0x6f00f400, "fmov v0.2d, #2.000000000000000000e+00"), + (0x4f00fc00, "fmov v0.8h, #2.000000000000000000e+00"), + (0x0f00fc00, "fmov v0.4h, #2.000000000000000000e+00"), + (0x4f04fc80, "fmov v0.8h, #-2.500000000000000000e+00"), + (0x2f00e420, "movi d0, #0xff"), + (0x2f0367e1, "mvni v1.2s, #0x7f, lsl #24"), + (0x2e023820, "ext v0.8b, v1.8b, v2.8b, #7"), + (0x6e024820, "ext v0.16b, v1.16b, v2.16b, #9"), + ]; + for (word, expected) in cases { + let instruction = decode(word).unwrap_or_else(|e| panic!("{word:08x}: {e:?}")); + assert_eq!(text(&instruction), expected, "{word:08x}"); + } + // Not instructions (`objdump`: `.inst ... ; undefined`) and so not decoded: an + // `ext .8b` index above 7, a modified immediate with bit 11 set outside the + // half-precision `fmov`, `fmov .2d` with `Q` clear. + for word in [0x2e164372, 0x0f000ced, 0x4f00af48, 0x2f00f420, 0x6f00fc20] { + assert!(decode(word).is_err(), "{word:08x}"); + } + } + + /// Fields beyond what the decoder produces must still print, not panic. + #[test] + fn extreme_fields_do_not_panic() { + use crate::simd::Op; + let ops = [ + Op::ShiftImm { + right: true, + signed: true, + size: u32::MAX, + q: true, + shift: u32::MAX, + rd: 255, + rn: 255, + }, + Op::ShiftLong { + signed: false, + size: 7, + shift: 1, + upper: true, + rd: 255, + rn: 255, + }, + Op::ScalarDup { + size: 99, + index: u32::MAX, + rd: 255, + rn: 255, + }, + Op::Narrow { + kind: crate::simd::Narrow::Plain, + size: 3, + upper: true, + rd: 0, + rn: 0, + }, + ]; + for op in ops { + let _ = text(&Instruction::Simd(op)); + } + let _ = text(&Instruction::SimdAlu(SimdAlu { + op: SimdAluOp::UAddLV, + rd: 255, + rn: 255, + rm: 255, + size: 255, + q: true, + })); + let _ = text(&Instruction::SimdDupElement(SimdDupElement { + rd: 255, + rn: 255, + esize: 255, + index: 255, + q: false, + })); + let _ = text(&Instruction::SimdInsert(SimdInsert { + rd: 255, + rn: 255, + esize: 255, + index: 255, + })); + let _ = text(&Instruction::SimdExtract(SimdExtract { + rd: 255, + rn: 255, + esize: 255, + index: 255, + })); + let _ = text(&Instruction::SimdTable(SimdTable { + rd: 255, + rn: 255, + rm: 255, + len: 255, + extend: true, + q: true, + })); + for low in [0, 1, u64::MAX, 0x0123_4567_89ab_cdef, 0x8000_0000_8000_0000] { + for combine in [ + ImmCombine::Set, + ImmCombine::SetInverted, + ImmCombine::Or, + ImmCombine::AndNot, + ] { + for form in [ + ImmForm::Byte, + ImmForm::Half { shift: 255 }, + ImmForm::Word { shift: 255 }, + ImmForm::WordMask { shift: 255 }, + ImmForm::Double, + ImmForm::FmovHalf, + ImmForm::FmovSingle, + ImmForm::FmovDouble, + ] { + for q in [false, true] { + let _ = text(&Instruction::SimdImm(SimdImmediate { + rd: 255, + imm8: 255, + form, + low, + high: if q { low } else { 0 }, + combine, + q, + })); + } + } + } + } + } +} diff --git a/src/isa/aarch64/src/fp.rs b/src/isa/aarch64/src/fp.rs new file mode 100644 index 0000000..3e21b4b --- /dev/null +++ b/src/isa/aarch64/src/fp.rs @@ -0,0 +1,358 @@ +//! Scalar floating point (single and double precision) and the vector `fabs`/`fneg` +//! forms: the decoded shape of every word [`decode`] recognises. +//! +//! Half precision is not decoded here (`type` 3), nor is anything with a reserved +//! `type`; those words are not recognised. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Precision { + Single, + Double, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Unary { + Move, + Abs, + Neg, + Sqrt, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Binary { + Mul, + Div, + Add, + Sub, + Max, + Min, + MaxNum, + MinNum, + Nmul, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Fused { + Madd, + Msub, + Nmadd, + Nmsub, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + Unary { + kind: Unary, + p: Precision, + rd: u8, + rn: u8, + }, + /// `fcvt`: between single and double; `p` is the source precision. + Convert { + p: Precision, + rd: u8, + rn: u8, + }, + Binary { + kind: Binary, + p: Precision, + rd: u8, + rn: u8, + rm: u8, + }, + Fused { + kind: Fused, + p: Precision, + rd: u8, + rn: u8, + rm: u8, + ra: u8, + }, + /// `fcmp`/`fcmpe`; no `rm` compares with zero. `signaling` is `fcmpe`. + Compare { + p: Precision, + rn: u8, + rm: Option, + signaling: bool, + }, + CondCompare { + p: Precision, + rn: u8, + rm: u8, + cond: u32, + nzcv: u32, + }, + Select { + p: Precision, + rd: u8, + rn: u8, + rm: u8, + cond: u32, + }, + Immediate { + p: Precision, + rd: u8, + imm8: u64, + }, + /// `scvtf`/`ucvtf` from a general register. + IntToFloat { + signed: bool, + p: Precision, + wide: bool, + rd: u8, + rn: u8, + }, + /// `fcvtzs`/`fcvtzu` to a general register. + FloatToInt { + signed: bool, + p: Precision, + wide: bool, + rd: u8, + rn: u8, + }, + /// `scvtf`/`ucvtf` between SIMD&FP registers (the source holds an integer). + ScalarIntToFloat { + signed: bool, + p: Precision, + rd: u8, + rn: u8, + }, + /// `fcvtzs`/`fcvtzu` between SIMD&FP registers. + ScalarFloatToInt { + signed: bool, + p: Precision, + rd: u8, + rn: u8, + }, + /// Vector `fabs`/`fneg`. + VectorSign { + negate: bool, + double: bool, + q: bool, + rd: u8, + rn: u8, + }, +} + +fn precision(ty: u32) -> Option { + match ty { + 0 => Some(Precision::Single), + 1 => Some(Precision::Double), + // 3 is half precision, which is not modelled; 2 is reserved. + _ => None, + } +} + +/// Decode `word` as a floating-point instruction, or `None` when it is not one. +pub fn decode(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = ((word >> 16) & 31) as u8; + let ty = (word >> 22) & 3; + // Scalar floating point: `M=0 S=0 01111 0 type 1 ...`. + if word & 0xff20_0c00 == 0x1e20_0800 { + let kind = match (word >> 12) & 15 { + 0 => Binary::Mul, + 1 => Binary::Div, + 2 => Binary::Add, + 3 => Binary::Sub, + 4 => Binary::Max, + 5 => Binary::Min, + 6 => Binary::MaxNum, + 7 => Binary::MinNum, + 8 => Binary::Nmul, + _ => return None, + }; + return Some(Op::Binary { + kind, + p: precision(ty)?, + rd, + rn, + rm, + }); + } + // One-source: bits 14:10 = 10000; the opcode (bits 20:15) includes bit 15. + if word & 0xff20_7c00 == 0x1e20_4000 { + let p = precision(ty)?; + let kind = match (word >> 15) & 0x3f { + 0 => Unary::Move, + 1 => Unary::Abs, + 2 => Unary::Neg, + 3 => Unary::Sqrt, + // `fcvt` to single from double, to double from single. + 4 if p == Precision::Double => return Some(Op::Convert { p, rd, rn }), + 5 if p == Precision::Single => return Some(Op::Convert { p, rd, rn }), + _ => return None, + }; + return Some(Op::Unary { kind, p, rd, rn }); + } + if word & 0xff20_fc07 == 0x1e20_2000 { + let zero = word & 8 != 0; + return Some(Op::Compare { + p: precision(ty)?, + rn, + rm: (!zero).then_some(rm), + signaling: word & 0x10 != 0, + }); + } + if word & 0xff20_0c10 == 0x1e20_0400 { + return Some(Op::CondCompare { + p: precision(ty)?, + rn, + rm, + cond: (word >> 12) & 15, + nzcv: word & 15, + }); + } + if word & 0xff20_0c00 == 0x1e20_0c00 { + return Some(Op::Select { + p: precision(ty)?, + rd, + rn, + rm, + cond: (word >> 12) & 15, + }); + } + if word & 0xff20_1fe0 == 0x1e20_1000 { + return Some(Op::Immediate { + p: precision(ty)?, + rd, + imm8: u64::from((word >> 13) & 0xff), + }); + } + if word & 0xff00_0000 == 0x1f00_0000 { + let kind = match ((word >> 21) & 1, (word >> 15) & 1) { + (0, 0) => Fused::Madd, + (0, _) => Fused::Msub, + (_, 0) => Fused::Nmadd, + _ => Fused::Nmsub, + }; + return Some(Op::Fused { + kind, + p: precision(ty)?, + rd, + rn, + rm, + ra: ((word >> 10) & 31) as u8, + }); + } + // Conversions between floating point and general registers. The `fmov` forms + // (rmode 0, opcode 6 and 7) are `SimdFmov`. + if word & 0x7f20_fc00 == 0x1e20_0000 { + let (p, wide) = (precision(ty)?, word & 0x8000_0000 != 0); + return match ((word >> 19) & 3, (word >> 16) & 7) { + (0, 2) => Some(Op::IntToFloat { + signed: true, + p, + wide, + rd, + rn, + }), + (0, 3) => Some(Op::IntToFloat { + signed: false, + p, + wide, + rd, + rn, + }), + (3, 0) => Some(Op::FloatToInt { + signed: true, + p, + wide, + rd, + rn, + }), + (3, 1) => Some(Op::FloatToInt { + signed: false, + p, + wide, + rd, + rn, + }), + _ => None, + }; + } + // The same conversions between SIMD&FP registers: bit 22 is `sz`. + let p = if word & 0x0040_0000 != 0 { + Precision::Double + } else { + Precision::Single + }; + match word & 0xffbf_fc00 { + 0x5e21_d800 => { + return Some(Op::ScalarIntToFloat { + signed: true, + p, + rd, + rn, + }); + } + 0x7e21_d800 => { + return Some(Op::ScalarIntToFloat { + signed: false, + p, + rd, + rn, + }); + } + 0x5ea1_b800 => { + return Some(Op::ScalarFloatToInt { + signed: true, + p, + rd, + rn, + }); + } + 0x7ea1_b800 => { + return Some(Op::ScalarFloatToInt { + signed: false, + p, + rd, + rn, + }); + } + _ => {} + } + // Vector `fabs` (U=0) and `fneg` (U=1). + if word & 0x9fbf_fc00 == 0x0ea0_f800 { + let (q, double) = (word & 0x4000_0000 != 0, word & 0x0040_0000 != 0); + // `.1d` is reserved. + if double && !q { + return None; + } + return Some(Op::VectorSign { + negate: word & 0x2000_0000 != 0, + double, + q, + rd, + rn, + }); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_encodings_are_not_claimed() { + // fmov d0, x1 and fmov x0, d1 are `SimdFmov`, not conversions. + assert_eq!(decode(0x9e670020), None); + assert_eq!(decode(0x9e660020), None); + // Half precision (type 3) and reserved type 2. + assert_eq!(decode(0x1ee22820), None); + assert_eq!(decode(0x1ea22820), None); + // fneg v0.1d is reserved. + assert_eq!(decode(0x2ee0f820), None); + assert!(decode(0x1e620820).is_some()); + } + + #[test] + fn compare_signaling() { + let cmp = |word| match decode(word) { + Some(Op::Compare { rm, signaling, .. }) => (rm, signaling), + other => panic!("{other:?}"), + }; + assert_eq!(cmp(0x1e222020), (Some(2), false)); // fcmp s1, s2 + assert_eq!(cmp(0x1e222030), (Some(2), true)); // fcmpe s1, s2 + assert_eq!(cmp(0x1e602008), (None, false)); // fcmp d0, #0.0 + assert_eq!(cmp(0x1e202038), (None, true)); // fcmpe s1, #0.0 + } +} diff --git a/src/isa/aarch64/src/gxf.rs b/src/isa/aarch64/src/gxf.rs new file mode 100644 index 0000000..a17d32d --- /dev/null +++ b/src/isa/aarch64/src/gxf.rs @@ -0,0 +1,41 @@ +//! Guarded execution (GXF): `genter` and `gexit`, the Apple instructions that move +//! between EL and the guarded levels GL1 and GL2 (Asahi Linux documentation, "Apple +//! Proprietary Instructions" and "SPRR and GXF"). + +/// `gexit`: leave guarded mode. +const GEXIT: u32 = 0x0020_1400; +/// `genter` with its `imm5` in bits 4:0 (stored in `ESR_GLx[5:0]`). +const GENTER: u32 = 0x0020_1420; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + Gexit, + Genter { imm5: u8 }, +} + +/// Decode `word` as `genter` or `gexit`, or `None` when it is neither. +pub fn decode(word: u32) -> Option { + if word == GEXIT { + Some(Op::Gexit) + } else if word & !0x1f == GENTER { + Some(Op::Genter { + imm5: (word & 0x1f) as u8, + }) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn genter_and_gexit_are_recognised_with_any_genter_immediate() { + assert_eq!(decode(GEXIT), Some(Op::Gexit)); + assert_eq!(decode(GENTER), Some(Op::Genter { imm5: 0 })); + assert_eq!(decode(GENTER | 0x1f), Some(Op::Genter { imm5: 31 })); + assert_eq!(decode(0xd503_201f), None); + assert_eq!(decode(GEXIT + 4), None); + } +} diff --git a/src/isa/aarch64/src/lib.rs b/src/isa/aarch64/src/lib.rs new file mode 100644 index 0000000..78bafde --- /dev/null +++ b/src/isa/aarch64/src/lib.rs @@ -0,0 +1,27 @@ +//! The AArch64 (A64) instruction set: what an instruction word *is*. +//! +//! This crate owns architectural knowledge and nothing else. It does not know +//! about guest state, memory, block translation, register allocation or +//! executable memory; consumers (disassembly, lifting, emulation) decode a word +//! once here and work from the result. +//! +//! * [`decode`] turns an instruction word into a structured +//! [`decode::Instruction`]. It is the only decoder: [`fp`], [`simd`], [`pauth`] +//! and [`gxf`] hold the instruction families it delegates to. +//! * [`format`] prints a decoded instruction as assembly text (`objdump` spelling). +//! * [`flow`] answers the control-flow question for analysis. +//! * [`simd_struct`] describes SIMD structure loads and stores. + +#![no_std] + +#[cfg(test)] +extern crate std; + +pub mod decode; +pub mod flow; +pub mod format; +pub mod fp; +pub mod gxf; +pub mod pauth; +pub mod simd; +pub mod simd_struct; diff --git a/src/isa/aarch64/src/pauth.rs b/src/isa/aarch64/src/pauth.rs new file mode 100644 index 0000000..174c2d5 --- /dev/null +++ b/src/isa/aarch64/src/pauth.rs @@ -0,0 +1,354 @@ +//! Pointer authentication (FEAT_PAuth) instructions: the decoded shape of every +//! word [`decode`] recognises, including the hint-space forms (`PACIASP`, ...). +//! +//! The cipher and what authentication does to a pointer are the emulator's business. + +/// The key the instruction uses. `Ga` is the generic key, used only by `PACGA`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum KeyId { + Ia, + Ib, + Da, + Db, + Ga, +} + +impl KeyId { + /// The `SCTLR_EL1` bit that enables the key, if any. The generic key has none. + pub fn enable_bit(self) -> Option { + match self { + Self::Ia => Some(31), + Self::Ib => Some(30), + Self::Da => Some(27), + Self::Db => Some(13), + Self::Ga => None, + } + } + + /// The `keynumber` the architecture reports in an authentication failure. + pub fn number(self) -> u64 { + match self { + Self::Ia => 0, + Self::Ib => 1, + Self::Da => 2, + Self::Db => 3, + Self::Ga => 4, + } + } +} + +/// The modifier operand: zero, or a register where register 31 is `SP`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Modifier { + Zero, + Reg(u8), +} + +/// One PAuth instruction, decoded. +/// +/// `hint` marks the hint-space spellings (`PACIASP`, `PACIA1716`, `PACIAZ`, `XPACLRI`, ...), +/// which execute as no-ops without PAuth; their register and modifier are fixed by the +/// opcode (`LR` signed with `SP`, `X17` with `X16`, `LR` with zero, `LR` stripped). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + /// `PACIA`, `PACIZA`, `PACDA`, ... and `PACIASP`, `PACIA1716`, `PACIAZ`, ... + Sign { + key: KeyId, + reg: u8, + modifier: Modifier, + hint: bool, + }, + /// `AUTIA`, `AUTIZA`, `AUTDA`, ... and `AUTIASP`, `AUTIA1716`, `AUTIAZ`, ... + Auth { + key: KeyId, + reg: u8, + modifier: Modifier, + hint: bool, + }, + /// `XPACI` (`data` clear), `XPACD` (`data` set) and the hint `XPACLRI`. + Strip { + data: bool, + reg: u8, + hint: bool, + }, + Pacga { + rd: u8, + rn: u8, + modifier: Modifier, + }, + /// `BRAA`, `BRAB`, `BLRAA`, `BLRAB` and their zero-modifier forms. + Branch { + key: KeyId, + link: bool, + target: u8, + modifier: Modifier, + }, + /// `RETAA` and `RETAB`: return to `LR`, authenticated with `SP`. + Return { + key: KeyId, + }, + /// `ERETAA` and `ERETAB`: return from the exception, authenticating `ELR_EL1`. + ExceptionReturn { + key: KeyId, + }, +} + +/// The hint-space forms (`PACIASP` and friends), which are no-ops without PAuth. +fn hint(word: u32) -> Option { + if word & 0xffff_f01f != 0xd503_201f { + return None; + } + let hint = (word >> 5) & 0x7f; + // Bit 2 selects authenticate over sign, bit 1 the B key over the A key. + let key = if hint & 2 != 0 { KeyId::Ib } else { KeyId::Ia }; + let (reg, modifier) = match hint { + 7 => { + return Some(Op::Strip { + data: false, + reg: 30, + hint: true, + }); + } + // PACIA1716, PACIB1716, AUTIA1716, AUTIB1716. + 8 | 10 | 12 | 14 => (17, Modifier::Reg(16)), + // PAC/AUT{IA,IB}Z (even) and PAC/AUT{IA,IB}SP (odd). + 24..=31 if hint & 1 == 0 => (30, Modifier::Zero), + 24..=31 => (30, Modifier::Reg(31)), + _ => return None, + }; + Some(if hint & 4 != 0 { + Op::Auth { + key, + reg, + modifier, + hint: true, + } + } else { + Op::Sign { + key, + reg, + modifier, + hint: true, + } + }) +} + +/// Decode `word` as a pointer-authentication instruction, or `None` when it is not one. +pub fn decode(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = rd; + // Data-processing, one source: PAC and AUT (opcodes 0-15), and XPAC (16-17). + if word & 0xffff_0000 == 0xdac1_0000 { + let opcode = (word >> 10) & 0x3f; + let key = [KeyId::Ia, KeyId::Ib, KeyId::Da, KeyId::Db][(opcode & 3) as usize]; + // The zero-modifier forms have no modifier register: Rn must be 31. + let modifier = if opcode & 8 != 0 { + if rn != 31 { + return None; + } + Modifier::Zero + } else { + Modifier::Reg(rn) + }; + return match opcode { + 0..=15 if opcode & 4 == 0 => Some(Op::Sign { + key, + reg: rd, + modifier, + hint: false, + }), + 0..=15 => Some(Op::Auth { + key, + reg: rd, + modifier, + hint: false, + }), + 16 | 17 if rn == 31 => Some(Op::Strip { + data: opcode == 17, + reg: rd, + hint: false, + }), + _ => None, + }; + } + // Data-processing, two sources: PACGA Xd, Xn, Xm|SP. + if word & 0xffe0_fc00 == 0x9ac0_3000 { + let rm = ((word >> 16) & 31) as u8; + return Some(Op::Pacga { + rd, + rn, + modifier: Modifier::Reg(rm), + }); + } + // Branches with authentication. Bit 10 selects the B key; bit 11 is always set. + let b_key = if word & 0x400 != 0 { + KeyId::Ib + } else { + KeyId::Ia + }; + if word & 0xffff_f800 == 0xd71f_0800 { + return Some(Op::Branch { + key: b_key, + link: false, + target: rn, + modifier: Modifier::Reg(rm), + }); + } + if word & 0xffff_f81f == 0xd61f_081f { + return Some(Op::Branch { + key: b_key, + link: false, + target: rn, + modifier: Modifier::Zero, + }); + } + if word & 0xffff_f800 == 0xd73f_0800 { + return Some(Op::Branch { + key: b_key, + link: true, + target: rn, + modifier: Modifier::Reg(rm), + }); + } + if word & 0xffff_f81f == 0xd63f_081f { + return Some(Op::Branch { + key: b_key, + link: true, + target: rn, + modifier: Modifier::Zero, + }); + } + if word & 0xffff_fbff == 0xd65f_0bff { + return Some(Op::Return { key: b_key }); + } + if word & 0xffff_fbff == 0xd69f_0bff { + return Some(Op::ExceptionReturn { key: b_key }); + } + hint(word) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sign(key: KeyId, reg: u8, modifier: Modifier, hint: bool) -> Option { + Some(Op::Sign { + key, + reg, + modifier, + hint, + }) + } + + fn auth(key: KeyId, reg: u8, modifier: Modifier, hint: bool) -> Option { + Some(Op::Auth { + key, + reg, + modifier, + hint, + }) + } + + #[test] + fn the_instruction_encodings_decode_as_their_names() { + // Assembled with GNU as for armv8.3-a. + assert_eq!( + decode(0xdac1_0020), + sign(KeyId::Ia, 0, Modifier::Reg(1), false) + ); + assert_eq!( + decode(0xdac1_11ac), + auth(KeyId::Ia, 12, Modifier::Reg(13), false) + ); + assert_eq!( + decode(0xdac1_23f0), + sign(KeyId::Ia, 16, Modifier::Zero, false) + ); + assert_eq!( + decode(0xdac1_43f8), + Some(Op::Strip { + data: false, + reg: 24, + hint: false + }) + ); + assert_eq!( + decode(0xd71f_0c43), + Some(Op::Branch { + key: KeyId::Ib, + link: false, + target: 2, + modifier: Modifier::Reg(3) + }) + ); + assert_eq!( + decode(0xd63f_095f), + Some(Op::Branch { + key: KeyId::Ia, + link: true, + target: 10, + modifier: Modifier::Zero + }) + ); + assert_eq!(decode(0xd65f_0fff), Some(Op::Return { key: KeyId::Ib })); + assert_eq!( + decode(0xd69f_0bff), + Some(Op::ExceptionReturn { key: KeyId::Ia }) + ); + // Plain BR and ordinary one-source data processing are not PAuth. + assert_eq!(decode(0xd61f_0000), None); + assert_eq!(decode(0xdac0_0000), None); + } + + #[test] + fn xpaci_and_xpacd_are_distinct() { + // `xpaci x0`, `xpacd x0`, `xpaci x30`, `xpacd x30`. + let strip = |data, reg, hint| Some(Op::Strip { data, reg, hint }); + assert_eq!(decode(0xdac1_43e0), strip(false, 0, false)); + assert_eq!(decode(0xdac1_47e0), strip(true, 0, false)); + assert_eq!(decode(0xdac1_43fe), strip(false, 30, false)); + assert_eq!(decode(0xdac1_47fe), strip(true, 30, false)); + // The hint `xpaclri` is its own spelling of stripping LR. + assert_eq!(decode(0xd503_20ff), strip(false, 30, true)); + } + + #[test] + fn hint_space_forms_differ_from_the_register_forms() { + use KeyId::{Ia, Ib}; + use Modifier::{Reg, Zero}; + // paciaz, paciasp, pacibz, pacibsp, autiaz, autiasp, autibz, autibsp. + assert_eq!(decode(0xd503_231f), sign(Ia, 30, Zero, true)); + assert_eq!(decode(0xd503_233f), sign(Ia, 30, Reg(31), true)); + assert_eq!(decode(0xd503_235f), sign(Ib, 30, Zero, true)); + assert_eq!(decode(0xd503_237f), sign(Ib, 30, Reg(31), true)); + assert_eq!(decode(0xd503_239f), auth(Ia, 30, Zero, true)); + assert_eq!(decode(0xd503_23bf), auth(Ia, 30, Reg(31), true)); + assert_eq!(decode(0xd503_23df), auth(Ib, 30, Zero, true)); + assert_eq!(decode(0xd503_23ff), auth(Ib, 30, Reg(31), true)); + // pacia1716, pacib1716, autia1716, autib1716. + assert_eq!(decode(0xd503_211f), sign(Ia, 17, Reg(16), true)); + assert_eq!(decode(0xd503_215f), sign(Ib, 17, Reg(16), true)); + assert_eq!(decode(0xd503_219f), auth(Ia, 17, Reg(16), true)); + assert_eq!(decode(0xd503_21df), auth(Ib, 17, Reg(16), true)); + // The register forms with the same operands: `pacia x30, sp`, `pacia x17, x16`, + // `paciza x30`, `autia x30, sp`. + assert_eq!(decode(0xdac1_03fe), sign(Ia, 30, Reg(31), false)); + assert_eq!(decode(0xdac1_0211), sign(Ia, 17, Reg(16), false)); + assert_eq!(decode(0xdac1_23fe), sign(Ia, 30, Zero, false)); + assert_eq!(decode(0xdac1_13fe), auth(Ia, 30, Reg(31), false)); + // Other hint numbers are not PAuth. + assert_eq!(decode(0xd503_203f), None); + assert_eq!(decode(0xd503_213f), None); + assert_eq!(decode(0xd503_243f), None); + } + + #[test] + fn zero_modifier_forms_need_register_31() { + // `paciza x0` with Rn = 1 is unallocated (objdump: `.inst ... ; undefined`). + assert_eq!(decode(0xdac1_2020), None); + assert_eq!(decode(0xdac1_3020), None); + // XPAC likewise needs Rn = 31. + assert_eq!(decode(0xdac1_4020), None); + } +} diff --git a/src/isa/aarch64/src/simd.rs b/src/isa/aarch64/src/simd.rs new file mode 100644 index 0000000..83b462a --- /dev/null +++ b/src/isa/aarch64/src/simd.rs @@ -0,0 +1,441 @@ +//! Integer Advanced SIMD instructions the structured decoder has no form for: the +//! vector permutes (`uzp`, `zip`, `trn`), the two-register-miscellaneous group +//! (`rev`, `cnt`, `mvn`, compares against zero, `abs`/`neg`, the narrowing `xtn` +//! family and the pairwise-long adds), the three-registers-different long and wide +//! arithmetic (`uaddl`, `umull`, ...), immediate shifts, and the 64-bit scalar forms. +//! +//! [`decode`] classifies a word; executing it is the emulator's business. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Permute { + Uzp1, + Uzp2, + Zip1, + Zip2, + Trn1, + Trn2, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Zero { + Gt, + Ge, + Le, + Lt, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Narrow { + Plain, + SignedSat, + UnsignedSat, + SignedToUnsignedSat, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Long { + Add, + AddWide, + Sub, + SubWide, + Abd, + Mul, + Mla, + Mls, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Scalar { + Add, + Sub, + Gt, + Hi, + Ge, + Hs, + Eq, + Tst, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Op { + Permute { + kind: Permute, + size: u32, + q: bool, + rd: u8, + rn: u8, + rm: u8, + }, + /// `rev16`/`rev32`/`rev64`: reverse the lanes inside each group of `group` bits. + Reverse { + group: u32, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Count { + q: bool, + rd: u8, + rn: u8, + }, + Not { + q: bool, + rd: u8, + rn: u8, + }, + ReverseBits { + q: bool, + rd: u8, + rn: u8, + }, + CompareZero { + kind: Zero, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Abs { + negate: bool, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + /// `xtn`/`sqxtn`/`uqxtn`/`sqxtun`; `upper` is the `2` form (keeps the low half). + Narrow { + kind: Narrow, + size: u32, + upper: bool, + rd: u8, + rn: u8, + }, + /// `saddlp`/`uaddlp` and, with `accumulate`, `sadalp`/`uadalp`. + PairLong { + signed: bool, + accumulate: bool, + size: u32, + q: bool, + rd: u8, + rn: u8, + }, + Long { + kind: Long, + signed: bool, + size: u32, + upper: bool, + rd: u8, + rn: u8, + rm: u8, + }, + ScalarAddp { + rd: u8, + rn: u8, + }, + ScalarDup { + size: u32, + index: u32, + rd: u8, + rn: u8, + }, + ScalarThree { + kind: Scalar, + rd: u8, + rn: u8, + rm: u8, + }, + /// `shl`, and `sshr`/`ushr`, by an immediate. + ShiftImm { + right: bool, + signed: bool, + size: u32, + q: bool, + shift: u32, + rd: u8, + rn: u8, + }, + /// `sshll`/`ushll` (`sxtl`/`uxtl` at shift 0); `upper` is the `2` form. + ShiftLong { + signed: bool, + size: u32, + shift: u32, + upper: bool, + rd: u8, + rn: u8, + }, + /// `shrn`/`shrn2`. + ShiftNarrow { + size: u32, + shift: u32, + upper: bool, + rd: u8, + rn: u8, + }, +} + +/// Decode `word` as one of these SIMD instructions, or `None` when it is not one. +pub fn decode(word: u32) -> Option { + let rd = (word & 31) as u8; + let rn = ((word >> 5) & 31) as u8; + let rm = ((word >> 16) & 31) as u8; + let q = word & 0x4000_0000 != 0; + let u = word & 0x2000_0000 != 0; + let size = (word >> 22) & 3; + // Permute: bit 21 clear, bits 11:10 = 10, bits 14:12 pick the operation. + if word & 0xbf20_8c00 == 0x0e00_0800 { + let kind = match (word >> 12) & 7 { + 1 => Permute::Uzp1, + 2 => Permute::Trn1, + 3 => Permute::Zip1, + 5 => Permute::Uzp2, + 6 => Permute::Trn2, + 7 => Permute::Zip2, + _ => return None, + }; + // 64-bit lanes need the full vector. + if size == 3 && !q { + return None; + } + return Some(Op::Permute { + kind, + size, + q, + rd, + rn, + rm, + }); + } + // Two-register miscellaneous: bits 21:17 = 10000, bits 11:10 = 10. + if word & 0x9f3e_0c00 == 0x0e20_0800 { + let opcode = (word >> 12) & 0x1f; + let wide_needs_q = size != 3 || q; + return match (u, opcode) { + (false, 0) if size < 3 => Some(Op::Reverse { + group: 64, + size, + q, + rd, + rn, + }), + (true, 0) if size < 2 => Some(Op::Reverse { + group: 32, + size, + q, + rd, + rn, + }), + (false, 1) if size == 0 => Some(Op::Reverse { + group: 16, + size, + q, + rd, + rn, + }), + (false | true, 2) if size < 3 => Some(Op::PairLong { + signed: !u, + accumulate: false, + size, + q, + rd, + rn, + }), + (false | true, 6) if size < 3 => Some(Op::PairLong { + signed: !u, + accumulate: true, + size, + q, + rd, + rn, + }), + (false, 5) if size == 0 => Some(Op::Count { q, rd, rn }), + (true, 5) if size == 0 => Some(Op::Not { q, rd, rn }), + (true, 5) if size == 1 => Some(Op::ReverseBits { q, rd, rn }), + (false, 8) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Gt, + size, + q, + rd, + rn, + }), + (true, 8) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Ge, + size, + q, + rd, + rn, + }), + (true, 9) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Le, + size, + q, + rd, + rn, + }), + (false, 10) if wide_needs_q => Some(Op::CompareZero { + kind: Zero::Lt, + size, + q, + rd, + rn, + }), + (false, 11) if wide_needs_q => Some(Op::Abs { + negate: false, + size, + q, + rd, + rn, + }), + (true, 11) if wide_needs_q => Some(Op::Abs { + negate: true, + size, + q, + rd, + rn, + }), + (false, 0x12) if size < 3 => Some(Op::Narrow { + kind: Narrow::Plain, + size, + upper: q, + rd, + rn, + }), + (true, 0x12) if size < 3 => Some(Op::Narrow { + kind: Narrow::SignedToUnsignedSat, + size, + upper: q, + rd, + rn, + }), + (false, 0x14) if size < 3 => Some(Op::Narrow { + kind: Narrow::SignedSat, + size, + upper: q, + rd, + rn, + }), + (true, 0x14) if size < 3 => Some(Op::Narrow { + kind: Narrow::UnsignedSat, + size, + upper: q, + rd, + rn, + }), + // `cmeq #0` (U=0, opcode 9) is `SimdCompareZero`. + _ => None, + }; + } + // Three registers different: bit 21 set, bits 11:10 = 00. + if word & 0x9f20_0c00 == 0x0e20_0000 && size < 3 { + let kind = match (word >> 12) & 15 { + 0 => Long::Add, + 1 => Long::AddWide, + 2 => Long::Sub, + 3 => Long::SubWide, + 7 => Long::Abd, + 8 => Long::Mla, + 10 => Long::Mls, + 12 => Long::Mul, + _ => return None, + }; + return Some(Op::Long { + kind, + signed: !u, + size, + upper: q, + rd, + rn, + rm, + }); + } + // Shift by immediate: `immh` (bits 22:19) gives the element size and, with + // `immb`, the shift. `immh == 0` is the modified-immediate class instead. + if word & 0x9f80_0400 == 0x0f00_0400 && (word >> 19) & 15 != 0 { + let immh = (word >> 19) & 15; + let immhb = (word >> 16) & 0x7f; + // The position of the top set bit of immh: 0 for bytes up to 3 for doublewords. + let size = 31 - immh.leading_zeros(); + let esize = 8 << size; + return match (u, (word >> 11) & 0x1f) { + (false, 0b01010) if size < 3 || q => Some(Op::ShiftImm { + right: false, + signed: false, + size, + q, + shift: immhb - esize, + rd, + rn, + }), + (_, 0b00000) if size < 3 || q => Some(Op::ShiftImm { + right: true, + signed: !u, + size, + q, + shift: 2 * esize - immhb, + rd, + rn, + }), + (_, 0b10100) if size < 3 => Some(Op::ShiftLong { + signed: !u, + size, + shift: immhb - esize, + upper: q, + rd, + rn, + }), + (false, 0b10000) if size < 3 => Some(Op::ShiftNarrow { + size, + shift: 2 * esize - immhb, + upper: q, + rd, + rn, + }), + _ => None, + }; + } + // Scalar pairwise `addp Dd, Vn.2D`. + if word & 0xffff_fc00 == 0x5ef1_b800 { + return Some(Op::ScalarAddp { rd, rn }); + } + // Scalar `dup`: `mov Bd/Hd/Sd/Dd, Vn.T[index]`. + if word & 0xffe0_fc00 == 0x5e00_0400 { + let imm5 = (word >> 16) & 0x1f; + if imm5 == 0 || imm5.trailing_zeros() > 3 { + return None; + } + let size = imm5.trailing_zeros(); + return Some(Op::ScalarDup { + size, + index: imm5 >> (size + 1), + rd, + rn, + }); + } + // Scalar three-same, 64-bit only. + if word & 0xdf20_0400 == 0x5e20_0400 && size == 3 { + let kind = match (u, (word >> 11) & 0x1f) { + (false, 0x10) => Scalar::Add, + (true, 0x10) => Scalar::Sub, + (false, 0x06) => Scalar::Gt, + (true, 0x06) => Scalar::Hi, + (false, 0x07) => Scalar::Ge, + (true, 0x07) => Scalar::Hs, + (true, 0x11) => Scalar::Eq, + (false, 0x11) => Scalar::Tst, + _ => return None, + }; + return Some(Op::ScalarThree { kind, rd, rn, rm }); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classification_declines_what_it_does_not_implement() { + // cmeq #0 is `SimdCompareZero`; half-width lanes need Q for 64-bit lanes; + // `ins`, `umov` and the three-same group belong to other decoders. + assert_eq!(decode(0x0e209820), None); + assert_eq!(decode(0x0ee0b820), None, "abs .1d is reserved"); + assert_eq!(decode(0x2ee21c20), None, "bif is three-same"); + assert_eq!(decode(0x4e083c20), None, "umov"); + assert!(decode(0x4e821820).is_some()); + } +} diff --git a/src/isa/aarch64/src/simd_struct.rs b/src/isa/aarch64/src/simd_struct.rs new file mode 100644 index 0000000..4f516c0 --- /dev/null +++ b/src/isa/aarch64/src/simd_struct.rs @@ -0,0 +1,51 @@ +//! SIMD structure loads and stores: `LD1`..`LD4`, `ST1`..`ST4`, their single-lane +//! forms and the replicating `LD1R`..`LD4R`. +//! +//! Decode describes one in a [`StructDesc`]. The emulator copies the descriptor into +//! its vCPU state and runs the access against its own memory, one element at a time. + +/// Which elements of the registers a structure access touches. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(u8)] +pub enum Shape { + /// Whole vectors: `ld1 {v0.4s - v3.4s}` and the interleaved `ld2`..`ld4`. + #[default] + Multiple, + /// One element of each register: `ld1 {v0.s}[1]`. Other lanes are kept. + Lane, + /// One element broadcast to every lane: `ld1r`. Loads only. + Replicate, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C)] +pub struct StructDesc { + pub store: bool, + pub shape: Shape, + /// Registers touched, starting at `rt` and wrapping from v31 to v0. + pub registers: u8, + /// `ld2`..`ld4`: consecutive memory elements alternate between the registers. + /// Otherwise (`ld1`) each register is a contiguous block. + pub interleaved: bool, + /// Element size in bytes: 1, 2, 4 or 8. + pub esize: u8, + /// Element index of a [`Shape::Lane`] access. + pub lane: u8, + /// A 128-bit vector (otherwise 64-bit, with the upper half zeroed on loads). + pub q: bool, + pub rt: u8, +} + +impl StructDesc { + pub fn vector_bytes(&self) -> u8 { + if self.q { 16 } else { 8 } + } + /// Bytes of memory the access spans; the post-index immediate advances by this. + pub fn bytes(&self) -> u64 { + let registers = u64::from(self.registers); + match self.shape { + Shape::Multiple => registers * u64::from(self.vector_bytes()), + Shape::Lane | Shape::Replicate => registers * u64::from(self.esize), + } + } +} diff --git a/src/lift/aarch64/Cargo.toml b/src/lift/aarch64/Cargo.toml new file mode 100644 index 0000000..865dbe8 --- /dev/null +++ b/src/lift/aarch64/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "volt-lift-aarch64" +version.workspace = true +edition.workspace = true +license.workspace = true + +[lib] +path = "src/lib.rs" + +[dependencies] +volt-ir.workspace = true +volt-isa-aarch64.workspace = true + +[dev-dependencies] +volt-target = { path = "../../target", default-features = false, features = ["aarch64"] } + +[lints] +workspace = true diff --git a/src/lift/aarch64/src/builder.rs b/src/lift/aarch64/src/builder.rs new file mode 100644 index 0000000..41df9ee --- /dev/null +++ b/src/lift/aarch64/src/builder.rs @@ -0,0 +1,287 @@ +use crate::{Environment, Writeback}; +use core::cell::{Cell, RefCell}; +use volt_ir::{ + function::{self as ir, BinOp as B, CmpOp as C, Function, Opcode, Value}, + types::{IntDesc, Type, TypeKind}, +}; +use volt_isa_aarch64::decode::{Addressing, Width}; + +/// Emits IR for one lifted function and knows where guest state lives. +/// +/// The builder owns the [`Function`] under construction, the block currently +/// being filled, the interned scalar types, and the state-pointer parameter +/// (`fn(state: *mut u8) -> u64`). Guest registers are addressed through the +/// offsets the [`Environment`] reported when the builder was created. +/// +/// Every method takes `&self`: emission order is the call order, and +/// [`Builder::set_block`] redirects subsequent emission, which is how an +/// environment builds fast and slow memory paths. +pub struct Builder { + f: RefCell, + /// Block currently being emitted into. + block: Cell, + state: Value, + /// 64-bit integer. + pub i64: Type, + /// 32-bit integer. + pub i32: Type, + /// 8-bit integer. + pub i8: Type, + /// One-bit comparison result. + pub boolean: Type, + /// Pointer; the type of the state pointer and of host addresses. + pub ptr: Type, + x: [usize; 31], + sp: usize, + flags: usize, + v: [usize; 32], +} +impl Builder { + pub(crate) fn new(env: &E) -> Self { + let mut f = Function::new(); + // A typical block is a few blocks and about a hundred instructions. + f.reserve(8, 128, 128); + let i64 = f.types.intern(TypeKind::Int(IntDesc { bits: 64 })); + let i32 = f.types.intern(TypeKind::Int(IntDesc { bits: 32 })); + let i8 = f.types.intern(TypeKind::Int(IntDesc { bits: 8 })); + let boolean = f.types.intern(TypeKind::Bool); + let ptr = f.types.ptr_global(); + let block = f.append_block(); + let state = f.append_block_param(block, ptr); + Self { + f: RefCell::new(f), + block: Cell::new(block), + state, + i64, + i32, + i8, + boolean, + ptr, + x: core::array::from_fn(|r| env.x_offset(r as u8)), + sp: env.sp_offset(), + flags: env.flags_offset(), + v: core::array::from_fn(|r| env.v_offset(r as u8)), + } + } + pub(crate) fn into_function(self) -> Function { + self.f.into_inner() + } + + // ---- guest state layout ---- + + /// The state pointer parameter of the function being built. + pub fn state(&self) -> Value { + self.state + } + /// Offset of general register `r`: `X0..=X30` for `r < 31`, `SP` for `r == 31`. + pub fn reg_at(&self, r: u8) -> usize { + if r == 31 { self.sp } else { self.x[r as usize] } + } + /// Offset of the 32-bit NZCV word. + pub fn flags_at(&self) -> usize { + self.flags + } + /// Offset of the low doubleword of vector register `r`. + pub fn v_at(&self, r: u8) -> usize { + self.v[r as usize] + } + + // ---- emission ---- + + /// Append `op` producing a value of type `t` to the current block. + pub fn emit(&self, t: Type, op: Opcode) -> Value { + self.f.borrow_mut().append_inst(self.block.get(), t, op) + } + /// An integer constant of type `t`. + pub fn k(&self, t: Type, n: u64) -> Value { + self.emit(t, Opcode::Iconst(n as i64)) + } + pub fn bin(&self, t: Type, op: B, a: Value, b: Value) -> Value { + self.emit(t, Opcode::Arith(ir::Arith { op, lhs: a, rhs: b })) + } + pub fn imm(&self, t: Type, op: B, a: Value, n: u64) -> Value { + // `ArithImm` instead of `Arith` over an `Iconst`: the backend encodes + // small immediates directly and `addrfold` only folds `ArithImm` adds + // into load/store addressing. + self.emit( + t, + Opcode::ArithImm(ir::ArithImm { + op, + lhs: a, + imm: n as i64, + }), + ) + } + /// Zero-extend or truncate `v` to `t`; `v` unchanged when it already has that type. + pub fn cv(&self, t: Type, v: Value) -> Value { + let from = self.f.borrow().value_type(v); + if from == t { + return v; + } + let kind = { + let f = self.f.borrow(); + let bits = |ty| match f.types.type_kind(ty) { + TypeKind::Int(d) => d.bits, + _ => 1, + }; + if bits(t) > bits(from) { + ir::ConvertKind::Zext + } else { + ir::ConvertKind::Trunc + } + }; + self.emit(t, Opcode::Convert(ir::Convert { value: v, kind })) + } + pub fn cmp(&self, op: C, a: Value, b: Value) -> Value { + self.emit( + self.boolean, + Opcode::Icmp(ir::Compare { op, lhs: a, rhs: b }), + ) + } + pub fn sel(&self, t: Type, c: Value, a: Value, b: Value) -> Value { + self.emit( + t, + Opcode::Select(ir::Select { + cond: c, + then: a, + else_: b, + }), + ) + } + /// The interned integer type of `bits` bits. + pub fn int(&self, bits: u16) -> Type { + self.f + .borrow_mut() + .types + .intern(TypeKind::Int(IntDesc { bits })) + } + pub(crate) fn ty(&self, w: Width) -> Type { + if w == Width::X64 { self.i64 } else { self.i32 } + } + + // ---- guest state access ---- + + /// Pointer to `state + at`. + pub fn addr(&self, at: usize) -> Value { + self.imm(self.ptr, B::Add, self.state, at as u64) + } + pub fn load_ptr(&self, t: Type, p: Value) -> Value { + self.emit( + t, + Opcode::Load(ir::Load { + ptr: p, + mem: ir::MemFlags::default(), + }), + ) + } + /// Load a `t` from `state + at`. + pub fn load(&self, t: Type, at: usize) -> Value { + self.load_ptr(t, self.addr(at)) + } + pub fn store_ptr(&self, p: Value, v: Value) { + self.f.borrow_mut().append_store(self.block.get(), v, p) + } + /// Store `v` to `state + at`. + pub fn store(&self, at: usize, v: Value) { + self.store_ptr(self.addr(at), v) + } + /// Store the 8-bit constant `v` to `state + at`. + pub fn byte(&self, at: usize, v: u64) { + self.store(at, self.k(self.i8, v)) + } + /// Read register `r` as a `t`. With `zero`, register 31 is `XZR` and reads + /// as zero; otherwise it is `SP`. + pub fn reg(&self, t: Type, r: u8, zero: bool) -> Value { + if zero && r >= 31 { + self.k(t, 0) + } else { + self.cv(t, self.load(self.i64, self.reg_at(r))) + } + } + /// Write `X`; a write to register 31 (`XZR`) is dropped. + pub fn put(&self, r: u8, v: Value) { + if r != 31 { + self.put_sp(r, v) + } + } + /// Write `X`, or `SP` for register 31. + pub fn put_sp(&self, r: u8, v: Value) { + self.store(self.reg_at(r), self.cv(self.i64, v)) + } + + // ---- control flow ---- + + /// Block currently being emitted into. + pub fn block(&self) -> ir::Block { + self.block.get() + } + /// Direct subsequent emission into `block`. + pub fn set_block(&self, block: ir::Block) { + self.block.set(block) + } + pub fn new_block(&self) -> ir::Block { + self.f.borrow_mut().append_block() + } + /// End the current block with a conditional branch. + pub fn branch_if(&self, c: Value, then: ir::Block, otherwise: ir::Block) { + self.f.borrow_mut().append_if( + self.block.get(), + c, + ir::EdgeDesc::bare(then), + ir::EdgeDesc::bare(otherwise), + ) + } + /// End the current block by returning `next_pc` from the function. An + /// environment's [`Environment::exit`] ends here; guest code never calls + /// this directly. + pub fn set_ret(&self, next_pc: Value) { + self.f + .borrow_mut() + .set_terminator(self.block.get(), ir::Terminator::Ret(ir::Ret::one(next_pc))) + } + + // ---- addressing ---- + + /// Effective address of a load/store with base `rn`, plus the base-register + /// update of a pre- or post-index mode. + /// + /// The update is returned instead of applied so each path can apply it its + /// own way once the access has succeeded: an access that faults must leave + /// the base register untouched so the retry sees it. + pub fn address(&self, rn: u8, a: Addressing) -> (Value, Option) { + let base = self.reg(self.i64, rn, false); + match a { + // The machines do not tell the unprivileged `ldtr`/`sttr` from a plain access. + Addressing::Offset(n) | Addressing::Unscaled(n) | Addressing::Unprivileged(n) => { + (self.imm(self.i64, B::Add, base, n as u64), None) + } + Addressing::PreIndex(n) => { + let address = self.imm(self.i64, B::Add, base, n as u64); + let writeback = Writeback { + register: rn, + value: address, + pre_index: true, + }; + (address, Some(writeback)) + } + Addressing::PostIndex(n) => { + let writeback = Writeback { + register: rn, + value: self.imm(self.i64, B::Add, base, n as u64), + pre_index: false, + }; + (base, Some(writeback)) + } + Addressing::Register { rm, extend, amount } => { + let v = self.extend(self.reg(self.i64, rm, true), extend); + let address = self.bin( + self.i64, + B::Add, + base, + self.imm(self.i64, B::Shl, v, u64::from(amount.unwrap_or(0))), + ); + (address, None) + } + } + } +} diff --git a/src/lift/aarch64/src/int.rs b/src/lift/aarch64/src/int.rs new file mode 100644 index 0000000..f23cd07 --- /dev/null +++ b/src/lift/aarch64/src/int.rs @@ -0,0 +1,383 @@ +//! Integer semantics: flags, conditions, shifts, extends, and the integer +//! instruction families too large to sit inline in the dispatcher. +use crate::Builder; +use volt_ir::{ + function::{BinOp as B, CmpOp as C, Value}, + types::Type, +}; +use volt_isa_aarch64::decode::*; + +impl Builder { + pub(crate) fn not(&self, v: Value) -> Value { + self.imm(self.boolean, B::BitXor, v, 1) + } + pub(crate) fn flag(&self, flags: Value, bit: u32) -> Value { + self.cmp( + C::Ne, + self.imm(self.i32, B::BitAnd, flags, 1 << bit), + self.k(self.i32, 0), + ) + } + pub(crate) fn condition(&self, c: Condition) -> Value { + let f = self.load(self.i32, self.flags_at()); + let n = self.flag(f, 31); + let z = self.flag(f, 30); + let carry = self.flag(f, 29); + let v = self.flag(f, 28); + let changed = self.bin(self.boolean, B::BitXor, n, v); + let ordered = self.not(changed); + let positive = self.not(z); + match c { + Condition::Eq => z, + Condition::Ne => positive, + Condition::Cs => carry, + Condition::Cc => self.not(carry), + Condition::Mi => n, + Condition::Pl => self.not(n), + Condition::Vs => v, + Condition::Vc => self.not(v), + Condition::Hi => self.bin(self.boolean, B::BitAnd, carry, positive), + Condition::Ls => self.not(self.bin(self.boolean, B::BitAnd, carry, positive)), + Condition::Ge => ordered, + Condition::Lt => changed, + Condition::Gt => self.bin(self.boolean, B::BitAnd, positive, ordered), + Condition::Le => self.not(self.bin(self.boolean, B::BitAnd, positive, ordered)), + Condition::Al | Condition::Nv => self.k(self.boolean, 1), + } + } + pub(crate) fn pack(&self, n: Value, z: Value, c: Value, v: Value) -> Value { + let mut out = self.k(self.i32, 0); + for (bit, shift) in [(n, 31), (z, 30), (c, 29), (v, 28)] { + out = self.bin( + self.i32, + B::BitOr, + out, + self.imm(self.i32, B::Shl, self.cv(self.i32, bit), shift), + ); + } + out + } + pub(crate) fn flags( + &self, + w: Width, + op: ArithOp, + a: Value, + b: Value, + r: Value, + carry: Option, + ) -> Value { + let t = self.ty(w); + let zero = self.k(t, 0); + let n = self.cmp( + C::Ne, + self.imm(t, B::BitAnd, r, 1u64 << ((w as u32) - 1)), + zero, + ); + let z = self.cmp(C::Eq, r, zero); + let c = carry.unwrap_or_else(|| { + if op == ArithOp::Sub { + self.cmp(C::Uge, a, b) + } else { + self.cmp(C::Ult, r, a) + } + }); + let an = self.cmp(C::Slt, a, zero); + let bn = self.cmp(C::Slt, b, zero); + let rn = self.cmp(C::Slt, r, zero); + let diff = self.bin(self.boolean, B::BitXor, an, bn); + let flip = self.bin(self.boolean, B::BitXor, an, rn); + let v = self.bin( + self.boolean, + B::BitAnd, + if op == ArithOp::Sub { + diff + } else { + self.not(diff) + }, + flip, + ); + self.pack(n, z, c, v) + } + pub(crate) fn logic_flags(&self, w: Width, r: Value) { + let t = self.ty(w); + let zero = self.k(t, 0); + let n = self.cmp( + C::Ne, + self.imm(t, B::BitAnd, r, 1u64 << ((w as u32) - 1)), + zero, + ); + let z = self.cmp(C::Eq, r, zero); + let no = self.k(self.boolean, 0); + self.store(self.flags_at(), self.pack(n, z, no, no)); + } + pub fn shift(&self, w: Width, v: Value, s: Shift, n: u64) -> Value { + if n == 0 { + return v; + } + let t = self.ty(w); + match s { + Shift::Lsl => self.imm(t, B::Shl, v, n), + Shift::Lsr => self.imm(t, B::Shr, v, n), + Shift::Asr => self.imm(t, B::Sar, v, n), + Shift::Ror => self.bin( + t, + B::BitOr, + self.imm(t, B::Shr, v, n), + self.imm(t, B::Shl, v, w as u64 - n), + ), + } + } + pub fn extend(&self, v: Value, e: Extend) -> Value { + match e { + Extend::Uxtx | Extend::Sxtx => v, + Extend::Uxtb => self.imm(self.i64, B::BitAnd, v, 0xff), + Extend::Uxth => self.imm(self.i64, B::BitAnd, v, 0xffff), + Extend::Uxtw => self.imm(self.i64, B::BitAnd, v, 0xffffffff), + Extend::Sxtb | Extend::Sxth | Extend::Sxtw => { + let n = match e { + Extend::Sxtb => 56, + Extend::Sxth => 48, + _ => 32, + }; + let raised = self.imm(self.i64, B::Shl, v, n); + self.shift(Width::X64, raised, Shift::Asr, n) + } + } + } +} +pub(crate) fn arithmetic_op(op: ArithOp) -> B { + if op == ArithOp::Sub { B::Sub } else { B::Add } +} +pub(crate) fn logic_op(op: LogicOp) -> B { + match op { + LogicOp::BitAnd => B::BitAnd, + LogicOp::BitOr => B::BitOr, + LogicOp::BitXor => B::BitXor, + } +} +pub(crate) fn low_mask(n: u8) -> u64 { + if n >= 64 { u64::MAX } else { (1u64 << n) - 1 } +} + +pub(crate) fn variable(b: &Builder, a: Variable) { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let rhs = b.reg(t, a.rm, true); + let bits = a.width as u64; + let r = match a.op { + VariableOp::Lsl | VariableOp::Lsr | VariableOp::Asr | VariableOp::Ror => { + let distance = b.imm(t, B::BitAnd, rhs, bits - 1); + match a.op { + VariableOp::Lsl => b.bin(t, B::Shl, lhs, distance), + VariableOp::Lsr => b.bin(t, B::Shr, lhs, distance), + VariableOp::Asr => b.bin(t, B::Sar, lhs, distance), + VariableOp::Ror => { + let back = b.imm( + t, + B::BitAnd, + b.bin(t, B::Sub, b.k(t, bits), distance), + bits - 1, + ); + b.bin( + t, + B::BitOr, + b.bin(t, B::Shr, lhs, distance), + b.bin(t, B::Shl, lhs, back), + ) + } + _ => unreachable!(), + } + } + VariableOp::Udiv | VariableOp::Sdiv => { + let zero = b.k(t, 0); + let one = b.k(t, 1); + let by_zero = b.cmp(C::Eq, rhs, zero); + if a.op == VariableOp::Udiv { + let safe = b.sel(t, by_zero, one, rhs); + b.sel(t, by_zero, zero, b.bin(t, B::UDiv, lhs, safe)) + } else { + let minus_one = b.cmp(C::Eq, rhs, b.k(t, u64::MAX)); + let safe = b.sel(t, minus_one, one, b.sel(t, by_zero, one, rhs)); + let quotient = b.bin(t, B::SDiv, lhs, safe); + let negated = b.bin(t, B::Sub, zero, lhs); + b.sel(t, by_zero, zero, b.sel(t, minus_one, negated, quotient)) + } + } + }; + if a.rd != 31 { + b.put(a.rd, r) + } +} +pub(crate) fn bitfield(b: &Builder, a: Bitfield) { + let t = b.ty(a.width); + let bits = a.width as u64; + let len = a.len as u64; + let lsb = a.lsb as u64; + let source = b.reg(t, a.rn, true); + let mask = low_mask(a.len); + let width_mask = if a.width == Width::X64 { + u64::MAX + } else { + u32::MAX as u64 + }; + let extract = |signed| { + let raised = b.imm(t, B::Shl, source, bits - lsb - len); + b.shift( + a.width, + raised, + if signed { Shift::Asr } else { Shift::Lsr }, + bits - len, + ) + }; + let r = match a.op { + BitfieldOp::Unsigned => { + if a.extract { + extract(false) + } else { + b.imm( + t, + B::Shl, + b.imm(t, B::BitAnd, source, mask & width_mask), + lsb, + ) + } + } + BitfieldOp::Signed => { + if a.extract { + extract(true) + } else { + let raised = b.imm(t, B::Shl, source, bits - len); + let extended = b.shift(a.width, raised, Shift::Asr, bits - len); + b.imm(t, B::Shl, extended, lsb) + } + } + BitfieldOp::Insert => { + let value = if a.extract { + extract(false) + } else { + b.imm( + t, + B::Shl, + b.imm(t, B::BitAnd, source, mask & width_mask), + lsb, + ) + }; + let place = if a.extract { 0 } else { lsb }; + let hole = (mask << place) & width_mask; + let kept = b.imm(t, B::BitAnd, b.reg(t, a.rd, true), !hole & width_mask); + b.bin(t, B::BitOr, kept, value) + } + }; + if a.rd != 31 { + b.put(a.rd, r) + } +} +fn repeated(pattern: u64, unit: u32, bits: u32) -> u64 { + let mut r = 0; + let mut at = 0; + while at < bits { + r |= pattern << at; + at += unit + } + r +} +fn swap(b: &Builder, t: Type, v: Value, mask: u64, n: u64) -> Value { + let low = b.imm(t, B::BitAnd, v, mask); + let high = b.imm(t, B::BitAnd, b.imm(t, B::Shr, v, n), mask); + b.bin(t, B::BitOr, b.imm(t, B::Shl, low, n), high) +} +pub(crate) fn unary(b: &Builder, a: Unary) { + let t = b.ty(a.width); + let bits = a.width as u32; + let mut v = b.reg(t, a.rn, true); + match a.op { + UnaryOp::Rev16 => v = swap(b, t, v, repeated(0xff, 16, bits), 8), + UnaryOp::Rev32 => { + v = swap(b, t, v, repeated(0xff, 16, bits), 8); + v = swap(b, t, v, repeated(0xffff, 32, bits), 16); + } + UnaryOp::Rev | UnaryOp::Rbit => { + if a.op == UnaryOp::Rbit { + for (mask, unit, n) in [(1, 2, 1), (3, 4, 2), (15, 8, 4)] { + v = swap(b, t, v, repeated(mask, unit, bits), n) + } + } + v = swap(b, t, v, repeated(0xff, 16, bits), 8); + v = swap(b, t, v, repeated(0xffff, 32, bits), 16); + if bits == 64 { + v = b.bin( + t, + B::BitOr, + b.imm(t, B::Shl, v, 32), + b.imm(t, B::Shr, v, 32), + ) + } + } + UnaryOp::Clz => { + let original = v; + let mut count = b.k(t, 0); + let mut step = bits / 2; + while step >= 1 { + let top = b.imm(t, B::Shr, v, (bits - step) as u64); + let empty = b.cmp(C::Eq, top, b.k(t, 0)); + v = b.sel(t, empty, b.imm(t, B::Shl, v, step as u64), v); + count = b.sel(t, empty, b.imm(t, B::Add, count, step as u64), count); + step /= 2; + } + v = b.sel( + t, + b.cmp(C::Eq, original, b.k(t, 0)), + b.k(t, bits as u64), + count, + ); + } + } + if a.rd != 31 { + b.put(a.rd, v) + } +} +pub(crate) fn high(b: &Builder, a: MultiplyHigh) { + let t = b.i64; + let x = b.reg(t, a.rn, true); + let y = b.reg(t, a.rm, true); + let xl = b.imm(t, B::BitAnd, x, 0xffffffff); + let yl = b.imm(t, B::BitAnd, y, 0xffffffff); + let xh = b.imm(t, B::Shr, x, 32); + let yh = b.imm(t, B::Shr, y, 32); + let ll = b.bin(t, B::Mul, xl, yl); + let lh = b.bin(t, B::Mul, xl, yh); + let hl = b.bin(t, B::Mul, xh, yl); + let hh = b.bin(t, B::Mul, xh, yh); + let middle = b.bin( + t, + B::Add, + b.bin( + t, + B::Add, + b.imm(t, B::Shr, ll, 32), + b.imm(t, B::BitAnd, lh, 0xffffffff), + ), + b.imm(t, B::BitAnd, hl, 0xffffffff), + ); + let mut high = b.bin( + t, + B::Add, + b.bin(t, B::Add, hh, b.imm(t, B::Shr, lh, 32)), + b.bin( + t, + B::Add, + b.imm(t, B::Shr, hl, 32), + b.imm(t, B::Shr, middle, 32), + ), + ); + if a.signed { + let zero = b.k(t, 0); + let xn = b.bin(t, B::Sub, zero, b.imm(t, B::Shr, x, 63)); + let yn = b.bin(t, B::Sub, zero, b.imm(t, B::Shr, y, 63)); + high = b.bin(t, B::Sub, high, b.bin(t, B::BitAnd, y, xn)); + high = b.bin(t, B::Sub, high, b.bin(t, B::BitAnd, x, yn)); + } + if a.rd != 31 { + b.put(a.rd, high) + } +} diff --git a/src/lift/aarch64/src/lib.rs b/src/lift/aarch64/src/lib.rs new file mode 100644 index 0000000..dcbdfa7 --- /dev/null +++ b/src/lift/aarch64/src/lib.rs @@ -0,0 +1,526 @@ +//! AArch64 instruction semantics lifted to [`volt_ir`], independent of any +//! particular machine. +//! +//! [`Lifter`] turns decoded [`Instruction`]s into IR over *guest state held in +//! memory*. Every lifted function has the signature +//! +//! ```text +//! fn(state: *mut u8) -> u64 +//! ``` +//! +//! It reads and writes guest registers at `state + offset`, and **returns the +//! next guest pc**. Which offsets, and everything that is not pure +//! architecture, belongs to the [`Environment`] the lifter is generic over: +//! +//! * **Layout.** [`Environment::x_offset`], [`sp_offset`](Environment::sp_offset), +//! [`flags_offset`](Environment::flags_offset) and +//! [`v_offset`](Environment::v_offset) say where `X0..=X30`, `SP`, the NZCV +//! word (`u32`, flags in bits 31..28) and the 128-bit vector registers (two +//! little-endian `u64`s) live. They are read once, when the [`Lifter`] is +//! created. +//! * **Memory.** Loads, stores, pairs, literals, exclusives, atomics and SIMD +//! structure accesses are handed to [`Environment::memory`] with the +//! effective address already computed. The environment decides how guest +//! memory is reached: a plain host load, a TLB probe, a call out, a trap. +//! * **Events.** System registers, `SVC`, `BRK`, `WFI`/`WFE`/`SEV`, barriers +//! that need machine action, TLB maintenance, `ERET`, `DC ZVA`, `AT`, +//! `CLREX` and cache operations go to [`Environment::event`]. +//! * **Exits.** Every path that leaves the function (branches, fall-through +//! at the end of a block, anything the environment ends itself) goes through +//! [`Environment::exit`], which by default returns the next pc. +//! +//! Everything else (integer ALU, flags, shifts, bitfields, multiplies, +//! conditional select/compare, `ADR`, branches and the register-only SIMD +//! families) is architecture and is lifted here. +//! +//! The lifter lifts instructions; the *caller* owns the block: it decodes, +//! decides where a block ends, and calls [`Lifter::exit`] when a block runs +//! off its end without a terminating instruction. Architectural branches come +//! from `Instruction::flow`; any further cut (an emulator ending a block at +//! every access it hands to the machine, say) is the caller's policy. An +//! instruction that falls through leaves the [`Builder`] positioned for the +//! next one. An instruction that does not (a branch, or a trap the environment +//! ends with [`Environment::exit`]) has already terminated the current block. +//! An environment must therefore end the block for every forwarded instruction +//! that the caller's policy treats as ending it. + +#![no_std] +extern crate alloc; +#[cfg(test)] +extern crate std; + +mod builder; +mod int; +mod simd; +#[cfg(test)] +mod tests; + +pub use builder::Builder; +use core::fmt; +use volt_ir::function::{BinOp as B, CmpOp as C, Function, Value}; +use volt_isa_aarch64::decode::*; + +/// A load, store or other guest memory instruction handed to +/// [`Environment::memory`]. +#[derive(Clone, Copy)] +pub struct MemoryAccess<'a> { + /// The decoded instruction: `Memory`, `Pair`, `Literal`, `SimdLiteral`, + /// `SimdMemory`, `SimdPair`, `Exclusive`, `Atomic` or `SimdStruct`. + pub insn: &'a Instruction, + /// The effective address, for the forms whose address is computed from the + /// addressing mode: `Memory`, `Pair`, `SimdMemory`, `SimdPair` (the base + /// for post-indexing, the updated base for pre-indexing), `Literal` and + /// `SimdLiteral` (a constant). `None` for `Exclusive`, `Atomic` and + /// `SimdStruct`, which address through a plain base register the + /// environment reads itself. + pub address: Option, + /// Base-register update of a pre- or post-index mode. The lifter does not + /// apply it: an access that can fault must leave the base untouched until + /// it has succeeded, so the environment writes it (with + /// [`Builder::put_sp`]) once it knows how. + pub writeback: Option, +} + +/// The base-register update of a pre- or post-indexed access. +#[derive(Clone, Copy, Debug)] +pub struct Writeback { + /// The base register; 31 is `SP`. + pub register: u8, + /// Its new value. + pub value: Value, + /// Pre-indexed: the access address is `value` itself. Otherwise the access + /// uses the old base and `value` is computed after it. + pub pre_index: bool, +} + +/// Why an instruction could not be lifted. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LiftError { + /// The [`Environment`] does not implement this instruction. + Unsupported, +} +impl fmt::Display for LiftError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Unsupported => f.write_str("instruction is not supported by the environment"), + } + } +} +impl core::error::Error for LiftError {} + +/// What a machine provides to the lifter: where guest state lives, how memory +/// is reached, and what instructions with machine effects do. See the +/// [crate documentation](crate). +pub trait Environment { + // ---- guest state layout (byte offsets from the state pointer) ---- + + /// Offset of `X` (`u64`), `r` in `0..=30`. + fn x_offset(&self, r: u8) -> usize; + /// Offset of the stack pointer (`u64`). + fn sp_offset(&self) -> usize; + /// Offset of the NZCV word (`u32`), flags in bits 31..28. + fn flags_offset(&self) -> usize; + /// Offset of vector register `V` (16 bytes: two little-endian `u64`s), `r` in `0..=31`. + fn v_offset(&self, r: u8) -> usize; + + // ---- hooks ---- + + /// Called once per guest instruction before it is lifted. An emulator + /// counts retired instructions here. + fn begin_instruction(&mut self, b: &Builder) { + let _ = b; + } + /// Called on every path that leaves the function with the next guest pc. + /// The default ends the current block with `Ret(next_pc)`; an environment + /// overrides it to do work on every exit first (for example flushing a + /// pending cycle counter) and then calls [`Builder::set_ret`]. + fn exit(&mut self, b: &Builder, next_pc: Value) { + b.set_ret(next_pc) + } + /// A load, store, exclusive, atomic or SIMD structure access at guest `pc`. + /// The environment emits whatever reaches memory; see [`MemoryAccess`] + /// for what the lifter has already computed. It must end the block (with + /// [`Self::exit`]) for instructions that terminate it. + fn memory(&mut self, b: &Builder, pc: u64, access: MemoryAccess<'_>) -> Result<(), LiftError>; + /// An instruction whose effect belongs to the machine: `System` + /// (`MRS`/`MSR`/`SYS`), `Svc`, `Brk`, `Wfi`, `Wfe`, `Sev`, `Sevl`, `Isb`, + /// `Tlbi`, `Eret`, `DcZva`, `AddressTranslate`, `Clrex`, `Psci` and + /// `CacheOp`. It receives the decoded instruction. + fn event(&mut self, b: &Builder, pc: u64, insn: &Instruction) -> Result<(), LiftError>; +} + +/// Lifts decoded AArch64 instructions into one [`Function`]. +pub struct Lifter { + b: Builder, + env: E, +} +impl Lifter { + /// Start a function with the layout `env` reports. + pub fn new(env: E) -> Self { + Self { + b: Builder::new(&env), + env, + } + } + /// The builder and the environment, for callers that emit IR of their own + /// between instructions (a block driver handling words it does not lift). + pub fn parts(&mut self) -> (&Builder, &mut E) { + (&self.b, &mut self.env) + } + /// The function ends at `next_pc`: the block fell off its end without a + /// terminating instruction. + pub fn exit(&mut self, next_pc: u64) { + let next = self.b.k(self.b.i64, next_pc); + self.env.exit(&self.b, next) + } + /// Finish: the function built so far, and the environment back. + pub fn into_parts(self) -> (Function, E) { + (self.b.into_function(), self.env) + } + /// Lift one decoded instruction at guest address `pc`. + pub fn lift(&mut self, pc: u64, insn: Instruction) -> Result<(), LiftError> { + let (b, env) = (&self.b, &mut self.env); + env.begin_instruction(b); + match insn { + Instruction::Movz(a) | Instruction::Movn(a) => { + let t = b.ty(a.width); + let bits = (a.immediate as u64) << a.shift; + let value = if matches!(insn, Instruction::Movn(_)) { + !bits + } else { + bits + }; + if a.rd != 31 { + b.put(a.rd, b.k(t, value)); + } + } + Instruction::Movk(a) => { + if a.rd != 31 { + let t = b.ty(a.width); + let old = b.reg(t, a.rd, false); + let kept = b.imm(t, B::BitAnd, old, !(0xffffu64 << a.shift)); + b.put( + a.rd, + b.imm(t, B::BitOr, kept, (a.immediate as u64) << a.shift), + ); + } + } + Instruction::ArithImm(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, false); + let rhs = b.k(t, u64::from(a.operand())); + let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); + if a.flags { + b.put(a.rd, r); + b.store(b.flags_at(), b.flags(a.width, a.op, lhs, rhs, r, None)); + } else { + b.put_sp(a.rd, r) + } + } + Instruction::ArithReg(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let rhs = b.shift(a.width, b.reg(t, a.rm, true), a.shift, a.amount as u64); + let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); + if a.rd != 31 { + b.put(a.rd, r) + } + if a.flags { + b.store(b.flags_at(), b.flags(a.width, a.op, lhs, rhs, r, None)); + } + } + Instruction::ArithExt(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, false); + let extended = b.extend(b.reg(b.i64, a.rm, true), a.extend); + let rhs = b.cv(t, b.imm(b.i64, B::Shl, extended, a.amount as u64)); + let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); + if a.flags { + b.put(a.rd, r); + b.store(b.flags_at(), b.flags(a.width, a.op, lhs, rhs, r, None)); + } else { + b.put_sp(a.rd, r) + } + } + Instruction::LogicReg(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let shifted = b.shift(a.width, b.reg(t, a.rm, true), a.shift, a.amount as u64); + let rhs = if a.invert { + b.imm(t, B::BitXor, shifted, u64::MAX) + } else { + shifted + }; + let r = b.bin(t, int::logic_op(a.op), lhs, rhs); + b.put(a.rd, r); + if a.flags { + b.logic_flags(a.width, r) + } + } + Instruction::LogicImm(a) => { + let t = b.ty(a.width); + let r = b.imm(t, int::logic_op(a.op), b.reg(t, a.rn, true), a.immediate); + if a.flags { + b.put(a.rd, r); + b.logic_flags(a.width, r) + } else { + b.put_sp(a.rd, r) + } + } + Instruction::ArithCarry(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let loaded = b.reg(t, a.rm, true); + let rhs = if a.op == ArithOp::Sub { + b.imm(t, B::BitXor, loaded, u64::MAX) + } else { + loaded + }; + let carry = b.cv( + t, + b.imm( + b.i32, + B::BitAnd, + b.imm(b.i32, B::Shr, b.load(b.i32, b.flags_at()), 29), + 1, + ), + ); + let partial = b.bin(t, B::Add, lhs, rhs); + let r = b.bin(t, B::Add, partial, carry); + if a.rd != 31 { + b.put(a.rd, r) + } + if a.flags { + let out = b.bin( + b.boolean, + B::BitOr, + b.cmp(C::Ult, partial, lhs), + b.cmp(C::Ult, r, partial), + ); + b.store( + b.flags_at(), + b.flags(a.width, ArithOp::Add, lhs, rhs, r, Some(out)), + ); + } + } + Instruction::CondCompare(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let rhs = match a.operand { + CompareOperand::Register(r) => b.reg(t, r, true), + CompareOperand::Immediate(n) => b.k(t, n as u64), + }; + let r = b.bin(t, int::arithmetic_op(a.op), lhs, rhs); + let flags = b.flags(a.width, a.op, lhs, rhs, r, None); + b.store( + b.flags_at(), + b.sel( + b.i32, + b.condition(a.cond), + flags, + b.k(b.i32, (a.nzcv as u64) << 28), + ), + ); + } + Instruction::Variable(a) => int::variable(b, a), + Instruction::Bitfield(a) => int::bitfield(b, a), + Instruction::Unary(a) => int::unary(b, a), + Instruction::Extract(a) => { + let t = b.ty(a.width); + let low = b.reg(t, a.rm, true); + let r = if a.lsb == 0 { + low + } else { + b.bin( + t, + B::BitOr, + b.imm(t, B::Shr, low, a.lsb as u64), + b.imm( + t, + B::Shl, + b.reg(t, a.rn, true), + a.width as u64 - a.lsb as u64, + ), + ) + }; + if a.rd != 31 { + b.put(a.rd, r) + } + } + Instruction::Mul(a) => { + let t = b.ty(a.width); + let p = b.bin(t, B::Mul, b.reg(t, a.rn, true), b.reg(t, a.rm, true)); + let r = b.bin(t, int::arithmetic_op(a.op), b.reg(t, a.ra, true), p); + if a.rd != 31 { + b.put(a.rd, r) + } + } + Instruction::MulLong(a) => { + let e = if a.signed { Extend::Sxtw } else { Extend::Uxtw }; + let p = b.bin( + b.i64, + B::Mul, + b.extend(b.reg(b.i64, a.rn, true), e), + b.extend(b.reg(b.i64, a.rm, true), e), + ); + let r = b.bin(b.i64, int::arithmetic_op(a.op), b.reg(b.i64, a.ra, true), p); + if a.rd != 31 { + b.put(a.rd, r) + } + } + Instruction::MulHigh(a) => int::high(b, a), + Instruction::Adr(a) => { + let here = if a.page { pc & 0xfffffffffffff000 } else { pc }; + b.put(a.rd, b.k(b.i64, here.wrapping_add(a.offset as u64))); + } + Instruction::Csel(a) => { + let t = b.ty(a.width); + let lhs = b.reg(t, a.rn, true); + let rhs = b.reg(t, a.rm, true); + let alt = match a.opc { + 0 => rhs, + 1 => b.imm(t, B::Add, rhs, 1), + 2 => b.imm(t, B::BitXor, rhs, u64::MAX), + 3 => b.bin(t, B::Sub, b.k(t, 0), rhs), + _ => unreachable!(), + }; + if a.rd != 31 { + b.put(a.rd, b.sel(t, b.condition(a.cond), lhs, alt)); + } + } + Instruction::TestBranch(a) => { + let t = b.ty(a.width); + let v = b.reg(t, a.rt, true); + let subject = if let Some(bit) = a.bit_index { + b.imm(t, B::BitAnd, v, 1u64 << bit) + } else { + v + }; + let taken = b.cmp(if a.negate { C::Ne } else { C::Eq }, subject, b.k(t, 0)); + branch(env, b, pc, a.offset, taken); + } + Instruction::BCond(a) => { + let taken = b.condition(a.cond); + branch(env, b, pc, a.offset, taken) + } + Instruction::B(offset) => { + let target = b.k(b.i64, pc.wrapping_add(offset as u64)); + env.exit(b, target) + } + Instruction::Call(a) => { + if a.link { + b.put(30, b.k(b.i64, pc.wrapping_add(4))) + } + let target = b.k(b.i64, pc.wrapping_add(a.target as u64)); + env.exit(b, target); + } + Instruction::Indirect(a) => { + let target = b.reg(b.i64, a.rn, false); + if a.kind == IndirectKind::Call { + b.put(30, b.k(b.i64, pc.wrapping_add(4))) + } + env.exit(b, target); + } + + Instruction::Memory(a) => { + let (address, writeback) = b.address(a.rn, a.addressing); + memory(env, b, pc, &insn, Some(address), writeback)? + } + Instruction::Pair(a) => { + let (address, writeback) = b.address(a.rn, a.addressing); + memory(env, b, pc, &insn, Some(address), writeback)? + } + Instruction::SimdMemory(a) => { + let (address, writeback) = b.address(a.rn, a.addressing); + memory(env, b, pc, &insn, Some(address), writeback)? + } + Instruction::SimdPair(a) => { + let (address, writeback) = b.address(a.rn, a.addressing); + memory(env, b, pc, &insn, Some(address), writeback)? + } + Instruction::Literal(a) => { + let address = b.k(b.i64, pc.wrapping_add(a.offset as u64)); + memory(env, b, pc, &insn, Some(address), None)? + } + Instruction::SimdLiteral(a) => { + let address = b.k(b.i64, pc.wrapping_add(a.offset as u64)); + memory(env, b, pc, &insn, Some(address), None)? + } + Instruction::Exclusive(_) | Instruction::Atomic(_) | Instruction::SimdStruct(_) => { + memory(env, b, pc, &insn, None, None)? + } + Instruction::SimdImm(a) => simd::imm(b, a), + Instruction::SimdInsert(a) => simd::insert(b, a), + Instruction::SimdInsertElement(a) => simd::insert_element(b, a), + Instruction::SimdExtract(a) => simd::extract(b, a), + Instruction::SimdDupElement(a) => simd::dup_element(b, a), + Instruction::SimdDup { rd, rn, esize, q } => simd::dup(b, rd, rn, esize, q), + Instruction::SimdExt(a) => simd::ext(b, a), + Instruction::SimdCompareZero(a) => simd::compare_zero(b, a), + Instruction::SimdTable(a) => simd::table(b, a), + Instruction::SimdFmov(a) => simd::fmov(b, a), + Instruction::SimdAlu(a) => simd::alu(b, a), + // Architecturally no-ops: nothing to emit. + Instruction::Nop + | Instruction::Hint(_) + | Instruction::Prefetch(_) + | Instruction::PrefetchLiteral(_) + | Instruction::RangePrefetch(_) + | Instruction::Dsb(_) + | Instruction::Dmb(_) + | Instruction::Yield => {} + Instruction::System(_) + | Instruction::Svc(_) + | Instruction::Brk(_) + | Instruction::Wfi + | Instruction::Wfe + | Instruction::Sev + | Instruction::Sevl + | Instruction::Isb(_) + | Instruction::Tlbi(_) + | Instruction::Eret + | Instruction::DcZva(_) + | Instruction::AddressTranslate(_) + | Instruction::Clrex(_) + | Instruction::Psci + | Instruction::CacheOp(_) => env.event(b, pc, &insn)?, + // Decoded by the same decoder, but the lifter has no semantics for these + // families: a machine that wants them runs them itself. + Instruction::Fp(_) + | Instruction::Simd(_) + | Instruction::Pauth(_) + | Instruction::Gxf(_) => { + return Err(LiftError::Unsupported); + } + } + Ok(()) + } +} +fn memory( + env: &mut E, + b: &Builder, + pc: u64, + insn: &Instruction, + address: Option, + writeback: Option, +) -> Result<(), LiftError> { + env.memory( + b, + pc, + MemoryAccess { + insn, + address, + writeback, + }, + ) +} +/// End the function at `pc + offset` when `taken`, otherwise at `pc + 4`. +fn branch(env: &mut E, b: &Builder, pc: u64, offset: i64, taken: Value) { + let next = b.sel( + b.i64, + taken, + b.k(b.i64, pc.wrapping_add(offset as u64)), + b.k(b.i64, pc.wrapping_add(4)), + ); + env.exit(b, next) +} diff --git a/src/lift/aarch64/src/simd.rs b/src/lift/aarch64/src/simd.rs new file mode 100644 index 0000000..5b9e0ec --- /dev/null +++ b/src/lift/aarch64/src/simd.rs @@ -0,0 +1,869 @@ +//! Register-only SIMD semantics: immediates, moves, lane insert/extract/dup, +//! `ext`, compare-against-zero, table lookup, `fmov` and the integer ALU. +//! +//! Every function works on the 128-bit vector registers in guest state as two +//! little-endian doublewords at [`Builder::v_at`]. +use crate::Builder; +use alloc::vec::Vec; +use volt_ir::function::{BinOp as B, CmpOp as C, Value}; +use volt_isa_aarch64::decode::*; +pub(crate) fn imm(b: &Builder, a: SimdImmediate) { + let at = b.v_at(a.rd); + match a.combine { + ImmCombine::Set | ImmCombine::SetInverted => { + b.store(at, b.k(b.i64, a.low)); + b.store(at + 8, b.k(b.i64, a.high)); + } + ImmCombine::Or | ImmCombine::AndNot => { + // A 64-bit form (`q == false`) also clears the upper half. + let (op, imm) = match a.combine { + ImmCombine::Or => (B::BitOr, a.low), + _ => (B::BitAnd, !a.low), + }; + b.store(at, b.imm(b.i64, op, b.load(b.i64, at), imm)); + b.store( + at + 8, + if a.q { + b.imm(b.i64, op, b.load(b.i64, at + 8), imm) + } else { + b.k(b.i64, 0) + }, + ); + } + } +} +pub(crate) fn insert(b: &Builder, a: SimdInsert) { + // Replace one lane of the destination with the low bits of a GPR. + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let at_d = b.v_at(a.rd); + let per_dw = (64 / ebits) as usize; + let word = a.index as usize / per_dw; + let shift = (a.index as usize % per_dw) as u64 * ebits; + let old = b.load(b.i64, at_d + word * 8); + let cleared = b.bin(b.i64, B::BitAnd, old, b.k(b.i64, !(emask << shift))); + let value = b.bin( + b.i64, + B::BitAnd, + b.reg(b.i64, a.rn, true), + b.k(b.i64, emask), + ); + b.store( + at_d + word * 8, + b.bin(b.i64, B::BitOr, cleared, b.imm(b.i64, B::Shl, value, shift)), + ); +} +pub(crate) fn insert_element(b: &Builder, a: SimdInsertElement) { + // Copy one lane of Vn over one lane of Vd. The source is read before + // the store, so `rd == rn` is fine. + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let per_dw = (64 / ebits) as usize; + let at_d = b.v_at(a.rd); + let at_n = b.v_at(a.rn); + let (src_word, src_shift) = ( + a.src as usize / per_dw, + (a.src as usize % per_dw) as u64 * ebits, + ); + let (dst_word, dst_shift) = ( + a.dst as usize / per_dw, + (a.dst as usize % per_dw) as u64 * ebits, + ); + let source = b.load(b.i64, at_n + src_word * 8); + let value = b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, source, src_shift), + emask, + ); + let old = b.load(b.i64, at_d + dst_word * 8); + let cleared = b.imm(b.i64, B::BitAnd, old, !(emask << dst_shift)); + b.store( + at_d + dst_word * 8, + b.bin( + b.i64, + B::BitOr, + cleared, + b.imm(b.i64, B::Shl, value, dst_shift), + ), + ); +} +pub(crate) fn extract(b: &Builder, a: SimdExtract) { + // Move one lane into the low bits of a GPR, zero-extended. + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let at_n = b.v_at(a.rn); + let per_dw = (64 / ebits) as usize; + let word = a.index as usize / per_dw; + let shift = (a.index as usize % per_dw) as u64 * ebits; + let source = b.load(b.i64, at_n + word * 8); + let value = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, source, shift), emask); + b.put(a.rd, value); +} +pub(crate) fn dup_element(b: &Builder, a: SimdDupElement) { + let ebits = (1u64 << a.esize) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let at_n = b.v_at(a.rn); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + // Locate the requested element across the 128-bit source. + let lane = a.index as usize; + let lanes_per_dw = (64 / ebits) as usize; + let src = if lane / lanes_per_dw == 0 { + vn_lo + } else { + vn_hi + }; + let shift = (lane % lanes_per_dw) as u64 * ebits; + let element = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, src, shift), emask); + // Replicate the element across the destination lanes. + let mut acc = b.k(b.i64, 0); + let lanes = (64 / ebits) as usize; + for lane in 0..lanes { + acc = b.bin( + b.i64, + B::BitOr, + acc, + b.imm(b.i64, B::Shl, element, lane as u64 * ebits), + ); + } + let at_d = b.v_at(a.rd); + b.store(at_d, acc); + b.store(at_d + 8, if a.q { acc } else { b.k(b.i64, 0) }); +} +pub(crate) fn dup(b: &Builder, rd: u8, rn: u8, esize: u8, q: bool) { + let r = b.reg(b.i64, rn, true); + let val64 = match esize { + 0 => { + let byte = b.imm(b.i64, B::BitAnd, r, 0xff); + b.bin(b.i64, B::Mul, byte, b.k(b.i64, 0x0101_0101_0101_0101)) + } + 1 => { + let h = b.imm(b.i64, B::BitAnd, r, 0xffff); + b.bin(b.i64, B::Mul, h, b.k(b.i64, 0x0001_0001_0001_0001)) + } + 2 => { + let w = b.imm(b.i64, B::BitAnd, r, 0xffff_ffff); + b.bin(b.i64, B::BitOr, w, b.imm(b.i64, B::Shl, w, 32)) + } + _ => r, + }; + let at = b.v_at(rd); + b.store(at, val64); + let hi = if q { val64 } else { b.k(b.i64, 0) }; + b.store(at + 8, hi); +} +pub(crate) fn ext(b: &Builder, a: SimdExt) { + let at_d = b.v_at(a.rd); + let at_n = b.v_at(a.rn); + let at_m = b.v_at(a.rm); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + let vm_lo = b.load(b.i64, at_m); + let vm_hi = b.load(b.i64, at_m + 8); + let (lo, hi) = if !a.q { + // `.8b`: a byte window over the low halves of `rm:rn`; the upper half clears. + let lo = if a.imm == 0 { + vn_lo + } else { + let s = a.imm as u64 * 8; + b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vn_lo, s), + b.imm(b.i64, B::Shl, vm_lo, 64 - s), + ) + }; + (lo, b.k(b.i64, 0)) + } else if a.imm == 0 { + (vn_lo, vn_hi) + } else if a.imm == 8 { + (vn_hi, vm_lo) + } else if a.imm < 8 { + let s = a.imm as u64 * 8; + let r = (8 - a.imm as u64) * 8; + let lo = b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vn_lo, s), + b.imm(b.i64, B::Shl, vn_hi, r), + ); + let hi = b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vn_hi, s), + b.imm(b.i64, B::Shl, vm_lo, r), + ); + (lo, hi) + } else { + let shift = a.imm as u64 - 8; + let s = shift * 8; + let r = (8 - shift) * 8; + let lo = b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vn_hi, s), + b.imm(b.i64, B::Shl, vm_lo, r), + ); + let hi = b.bin( + b.i64, + B::BitOr, + b.imm(b.i64, B::Shr, vm_lo, s), + b.imm(b.i64, B::Shl, vm_hi, r), + ); + (lo, hi) + }; + b.store(at_d, lo); + b.store(at_d + 8, hi); +} +pub(crate) fn compare_zero(b: &Builder, a: SimdCompareZero) { + let at_n = b.v_at(a.rn); + let at_d = b.v_at(a.rd); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + let ebits = (1u64 << a.size) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let halves = if a.q { 2 } else { 1 }; + for half in 0..2 { + if half >= halves { + b.store(at_d + half * 8, b.k(b.i64, 0)); + continue; + } + let v = if half == 0 { vn_lo } else { vn_hi }; + let mut acc = b.k(b.i64, 0); + let lanes = 8 >> a.size; + for lane in 0..lanes { + let shift = lane as u64 * ebits; + let field = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, v, shift), emask); + let is_zero = b.cmp(C::Eq, field, b.k(b.i64, 0)); + let bits = b.sel(b.i64, is_zero, b.k(b.i64, emask), b.k(b.i64, 0)); + acc = b.bin(b.i64, B::BitOr, acc, b.imm(b.i64, B::Shl, bits, shift)); + } + b.store(at_d + half * 8, acc); + } +} +pub(crate) fn table(b: &Builder, a: SimdTable) { + // Table bytes live in `len + 1` consecutive registers. Each index + // byte selects a table byte; out-of-range indices leave zero for + // `tbl` and keep the destination byte for `tbx`. + let at_m = b.v_at(a.rm); + let vm_lo = b.load(b.i64, at_m); + let vm_hi = b.load(b.i64, at_m + 8); + let regs = a.len as usize + 1; + let mut table = Vec::with_capacity(regs * 2); + for i in 0..regs { + let reg = (a.rn as usize + i) % 32; + let at = b.v_at(reg as u8); + table.push(b.load(b.i64, at)); + table.push(b.load(b.i64, at + 8)); + } + let at_d = b.v_at(a.rd); + let old_lo = if a.extend { + Some(b.load(b.i64, at_d)) + } else { + None + }; + let old_hi = if a.extend { + Some(b.load(b.i64, at_d + 8)) + } else { + None + }; + let halves = if a.q { 2 } else { 1 }; + let limit = (16 * regs) as u64; + for half in 0..2 { + if half >= halves { + b.store(at_d + half * 8, b.k(b.i64, 0)); + continue; + } + let indices = if half == 0 { vm_lo } else { vm_hi }; + let old = if half == 0 { old_lo } else { old_hi }; + let mut acc = b.k(b.i64, 0); + for byte in 0..8u64 { + let shift = byte * 8; + let index = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, indices, shift), 0xff); + let in_range = b.cmp(C::Ult, index, b.k(b.i64, limit)); + let mut value = b.k(b.i64, 0); + let sel = |c: Value, x: Value, y: Value| b.sel(b.i64, c, x, y); + for (slot, entry) in table.iter().enumerate() { + let here = b.cmp( + C::Eq, + b.bin(b.i64, B::Shr, index, b.k(b.i64, 3)), + b.k(b.i64, slot as u64), + ); + let byte_at = b.imm( + b.i64, + B::BitAnd, + b.bin( + b.i64, + B::Shr, + *entry, + b.bin( + b.i64, + B::Shl, + b.imm(b.i64, B::BitAnd, index, 7), + b.k(b.i64, 3), + ), + ), + 0xff, + ); + value = sel(b.bin(b.boolean, B::BitAnd, in_range, here), byte_at, value); + } + if a.extend { + value = sel( + in_range, + value, + b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, old.unwrap(), shift), + 0xff, + ), + ); + } + acc = b.bin(b.i64, B::BitOr, acc, b.imm(b.i64, B::Shl, value, shift)); + } + b.store(at_d + half * 8, acc); + } +} +pub(crate) fn fmov(b: &Builder, a: SimdFmov) { + // Pure bit move; no FP arithmetic involved. + // `fmov Vd.D[1], Xn` writes only the upper doubleword; `fmov Xd, Vn.D[1]` reads it. + let half = if a.upper { 8 } else { 0 }; + if a.to_fp { + let v = b.reg(b.i64, a.rn, true); + let bits = if a.double { + v + } else { + b.imm(b.i64, B::BitAnd, v, 0xffff_ffff) + }; + let at = b.v_at(a.rd); + b.store(at + half, bits); + if !a.upper { + b.store(at + 8, b.k(b.i64, 0)); + } + } else { + let at = b.v_at(a.rn); + let v = b.load(b.i64, at + half); + let bits = if a.double { + v + } else { + b.imm(b.i64, B::BitAnd, v, 0xffff_ffff) + }; + b.put(a.rd, bits); + } +} +pub(crate) fn alu(b: &Builder, a: SimdAlu) { + // Bitwise logical operations work on whole halves, lane width free. + if matches!( + a.op, + SimdAluOp::AddV | SimdAluOp::UMinV | SimdAluOp::UMaxV | SimdAluOp::UAddLV + ) { + // Horizontal add, minimum, maximum or widening add: reduce every + // lane of the source and store the result in lane 0, zeroing the + // rest. `UAddLV` keeps twice the lane width. + let ebits = (1u64 << a.size) * 8; + let mask_of = |bits: u64| { + if bits >= 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + } + }; + let emask = mask_of(ebits); + let result_mask = if a.op == SimdAluOp::UAddLV { + mask_of(ebits * 2) + } else { + emask + }; + let at_n = b.v_at(a.rn); + let at_d = b.v_at(a.rd); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + let halves = if a.q { 2 } else { 1 }; + let mut total = if a.op == SimdAluOp::UMinV { + b.k(b.i64, emask) + } else { + b.k(b.i64, 0) + }; + for half in 0..halves { + let v = if half == 0 { vn_lo } else { vn_hi }; + for lane in 0..(8 >> a.size) { + let shift = lane as u64 * ebits; + let field = b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, v, shift), emask); + total = match a.op { + SimdAluOp::UMinV => b.sel(b.i64, b.cmp(C::Ult, field, total), field, total), + SimdAluOp::UMaxV => b.sel(b.i64, b.cmp(C::Ugt, field, total), field, total), + _ => b.bin(b.i64, B::Add, total, field), + }; + } + } + b.store( + at_d, + b.bin(b.i64, B::BitAnd, total, b.k(b.i64, result_mask)), + ); + b.store(at_d + 8, b.k(b.i64, 0)); + } else if matches!( + a.op, + SimdAluOp::And + | SimdAluOp::Bic + | SimdAluOp::Orr + | SimdAluOp::Orn + | SimdAluOp::Eor + | SimdAluOp::Bsl + | SimdAluOp::Bit + | SimdAluOp::Bif + ) { + let at_n = b.v_at(a.rn); + let at_m = b.v_at(a.rm); + let at_d = b.v_at(a.rd); + for half in 0..2 { + if half == 1 && !a.q { + b.store(at_d + 8, b.k(b.i64, 0)); + continue; + } + let x = b.load(b.i64, at_n + half * 8); + let y = b.load(b.i64, at_m + half * 8); + let not = |v| b.bin(b.i64, B::BitXor, v, b.k(b.i64, u64::MAX)); + let r = match a.op { + SimdAluOp::And => b.bin(b.i64, B::BitAnd, x, y), + SimdAluOp::Bic => b.bin(b.i64, B::BitAnd, x, not(y)), + SimdAluOp::Orr => b.bin(b.i64, B::BitOr, x, y), + SimdAluOp::Orn => b.bin(b.i64, B::BitOr, x, not(y)), + SimdAluOp::Eor => b.bin(b.i64, B::BitXor, x, y), + // The destination is both a source mask and the + // merge base for BSL/BIT/BIF. + _ => { + let d = b.load(b.i64, at_d + half * 8); + match a.op { + SimdAluOp::Bsl => b.bin( + b.i64, + B::BitOr, + b.bin(b.i64, B::BitAnd, x, d), + b.bin(b.i64, B::BitAnd, y, not(d)), + ), + _ => { + let selected = match a.op { + SimdAluOp::Bit => { + b.bin(b.i64, B::BitAnd, b.bin(b.i64, B::BitXor, d, x), y) + } + _ => b.bin(b.i64, B::BitAnd, b.bin(b.i64, B::BitXor, d, x), not(y)), + }; + b.bin(b.i64, B::BitXor, d, selected) + } + } + } + }; + b.store(at_d + half * 8, r); + } + } else { + let at_n = b.v_at(a.rn); + let at_m = b.v_at(a.rm); + let at_d = b.v_at(a.rd); + let vn_lo = b.load(b.i64, at_n); + let vn_hi = b.load(b.i64, at_n + 8); + let vm_lo = b.load(b.i64, at_m); + let vm_hi = b.load(b.i64, at_m + 8); + let need_d = matches!(a.op, SimdAluOp::Mla | SimdAluOp::Mls); + // Pairwise forms consume adjacent lanes of the full source pair: + // lane 0/1 of Vn and Vm, then lane 2/3, and so on. + if matches!( + a.op, + SimdAluOp::UMaxP + | SimdAluOp::SMaxP + | SimdAluOp::UMinP + | SimdAluOp::SMinP + | SimdAluOp::AddP + ) { + let ebits = (1u64 << a.size) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let at_n = b.v_at(a.rn); + let at_m = b.v_at(a.rm); + let at_d = b.v_at(a.rd); + let ssel = |c: Value, x: Value, y: Value| b.sel(b.i64, c, x, y); + let sources = [ + b.load(b.i64, at_n), + b.load(b.i64, at_n + 8), + b.load(b.i64, at_m), + b.load(b.i64, at_m + 8), + ]; + // Each source contributes `per_source` lanes (a 64-bit vector for + // the D form, a 128-bit one for the Q form). Lane `2k` pairs with + // lane `2k + 1` and the result lands in result lane + // `which * reduced + k`: the reduction of Vn fills the low half of + // the result and Vm the high half. For the D form the whole result + // is 64 bits, so Vm's lanes land in bits 32..64. + let per_source = ((if a.q { 128 } else { 64 }) / ebits) as usize; + let reduced_lanes = per_source / 2; + let signed = matches!(a.op, SimdAluOp::SMaxP | SimdAluOp::SMinP); + let mut out = [b.k(b.i64, 0); 2]; + for (which, half) in [(0usize, 0usize), (2usize, 1usize)] { + for k in 0..reduced_lanes { + let lanes_per_dw = (64 / ebits) as usize; + let read = |lane: usize| { + let double = lane / lanes_per_dw; + let shift = (lane % lanes_per_dw) as u64 * ebits; + let raw = b.imm( + b.i64, + B::BitAnd, + b.imm(b.i64, B::Shr, sources[which + double], shift), + emask, + ); + if signed { + // Ordered as signed numbers: sign-extend the lane. + let up = 64 - ebits; + b.imm(b.i64, B::Sar, b.imm(b.i64, B::Shl, raw, up), up) + } else { + raw + } + }; + let lo = read(2 * k); + let hi = read(2 * k + 1); + let r = match a.op { + SimdAluOp::AddP => b.bin(b.i64, B::Add, lo, hi), + SimdAluOp::UMaxP => ssel(b.cmp(C::Ugt, lo, hi), lo, hi), + SimdAluOp::UMinP => ssel(b.cmp(C::Ult, lo, hi), lo, hi), + SimdAluOp::SMaxP => ssel(b.cmp(C::Sgt, lo, hi), lo, hi), + _ => ssel(b.cmp(C::Slt, lo, hi), lo, hi), + }; + let masked = b.bin(b.i64, B::BitAnd, r, b.k(b.i64, emask)); + // Result lane `index` sits at bit `index * ebits` of the + // 128-bit result. + let index = half * reduced_lanes + k; + let bit = index as u64 * ebits; + let slot = (bit / 64) as usize; + out[slot] = b.bin( + b.i64, + B::BitOr, + out[slot], + b.imm(b.i64, B::Shl, masked, bit % 64), + ); + } + } + b.store(at_d, out[0]); + b.store(at_d + 8, out[1]); + } else { + let vd_lo = if need_d { + Some(b.load(b.i64, at_d)) + } else { + None + }; + let vd_hi = if need_d { + Some(b.load(b.i64, at_d + 8)) + } else { + None + }; + let ebits = (1u64 << a.size) * 8; + let emask = if ebits == 64 { + u64::MAX + } else { + (1u64 << ebits) - 1 + }; + let smax = u64::MAX >> (65 - ebits); + let smin = !smax; + let halves = if a.q { 2 } else { 1 }; + for half in 0..2 { + if half >= halves { + b.store(at_d + half * 8, b.k(b.i64, 0)); + continue; + } + let vn = if half == 0 { vn_lo } else { vn_hi }; + let vm = if half == 0 { vm_lo } else { vm_hi }; + let vd = if half == 0 { vd_lo } else { vd_hi }; + let mut acc = b.k(b.i64, 0); + let lanes = 8 >> a.size; + for lane in 0..lanes { + let shift = lane as u64 * ebits; + let fld = |v| b.imm(b.i64, B::BitAnd, b.imm(b.i64, B::Shr, v, shift), emask); + let ext = |v: Value| { + let up = b.bin(b.i64, B::Shl, v, b.k(b.i64, 64 - ebits)); + b.imm(b.i64, B::Sar, up, 64 - ebits) + }; + let an = fld(vn); + let bn = fld(vm); + let sa = ext(an); + let sb = ext(bn); + let dn = vd.map(fld); + let ssel = |c: Value, x: Value, y: Value| b.sel(b.i64, c, x, y); + let ssat = |v: Value| { + let over = b.cmp(C::Sgt, v, b.k(b.i64, smax)); + let under = b.cmp(C::Slt, v, b.k(b.i64, smin)); + ssel(over, b.k(b.i64, smax), ssel(under, b.k(b.i64, smin), v)) + }; + let r = match a.op { + SimdAluOp::Add => b.bin(b.i64, B::Add, an, bn), + SimdAluOp::Sub => b.bin(b.i64, B::Sub, an, bn), + SimdAluOp::Mul => b.bin(b.i64, B::Mul, an, bn), + SimdAluOp::Mla => { + b.bin(b.i64, B::Add, dn.unwrap(), b.bin(b.i64, B::Mul, an, bn)) + } + SimdAluOp::Mls => { + b.bin(b.i64, B::Sub, dn.unwrap(), b.bin(b.i64, B::Mul, an, bn)) + } + SimdAluOp::CmEq => { + ssel(b.cmp(C::Eq, an, bn), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmGt => { + ssel(b.cmp(C::Sgt, sa, sb), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmGe => { + ssel(b.cmp(C::Sge, sa, sb), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmHi => { + ssel(b.cmp(C::Ugt, an, bn), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmHs => { + ssel(b.cmp(C::Uge, an, bn), b.k(b.i64, emask), b.k(b.i64, 0)) + } + SimdAluOp::CmTst => ssel( + b.cmp(C::Ne, b.bin(b.i64, B::BitAnd, an, bn), b.k(b.i64, 0)), + b.k(b.i64, emask), + b.k(b.i64, 0), + ), + SimdAluOp::UMin => ssel(b.cmp(C::Ult, an, bn), an, bn), + SimdAluOp::UMax => ssel(b.cmp(C::Ugt, an, bn), an, bn), + SimdAluOp::SMin => ssel(b.cmp(C::Slt, sa, sb), an, bn), + SimdAluOp::SMax => ssel(b.cmp(C::Sgt, sa, sb), an, bn), + SimdAluOp::UAbd => { + let d = b.bin(b.i64, B::Sub, an, bn); + ssel( + b.cmp(C::Uge, an, bn), + d, + b.bin(b.i64, B::Sub, b.k(b.i64, 0), d), + ) + } + SimdAluOp::SAbd => { + let d = b.bin(b.i64, B::Sub, sa, sb); + let neg = b.bin(b.i64, B::Sub, b.k(b.i64, 0), d); + ssel(b.cmp(C::Sge, sa, sb), d, neg) + } + SimdAluOp::SQAdd => ssat(b.bin(b.i64, B::Add, sa, sb)), + SimdAluOp::UQAdd => { + let s = b.bin(b.i64, B::Add, an, bn); + let carry = b.cmp(C::Ult, s, an); + let big = b.cmp(C::Ugt, s, b.k(b.i64, emask)); + ssel(b.bin(b.i64, B::BitOr, carry, big), b.k(b.i64, emask), s) + } + SimdAluOp::SQSub => ssat(b.bin(b.i64, B::Sub, sa, sb)), + SimdAluOp::UQSub => { + let borrow = b.cmp(C::Ugt, bn, an); + ssel(borrow, b.k(b.i64, 0), b.bin(b.i64, B::Sub, an, bn)) + } + SimdAluOp::SHAdd + | SimdAluOp::UHAdd + | SimdAluOp::SHSub + | SimdAluOp::UHSub + | SimdAluOp::SRHAdd + | SimdAluOp::URHAdd => { + let signed_op = matches!( + a.op, + SimdAluOp::SHAdd | SimdAluOp::SHSub | SimdAluOp::SRHAdd + ); + // Signed forms operate on sign-extended lanes, which + // keeps the halving shift meaningful for negatives. + let (op_a, op_b) = if signed_op { (sa, sb) } else { (an, bn) }; + let shr = |v: Value| { + if signed_op { + b.imm(b.i64, B::Sar, v, 1) + } else { + b.imm(b.i64, B::Shr, v, 1) + } + }; + let k = b.bin(b.i64, B::Sub, shr(op_a), shr(op_b)); + let k = if matches!( + a.op, + SimdAluOp::SHAdd + | SimdAluOp::UHAdd + | SimdAluOp::SRHAdd + | SimdAluOp::URHAdd + ) { + b.bin(b.i64, B::Add, shr(op_a), shr(op_b)) + } else { + k + }; + let a0 = b.bin(b.i64, B::BitAnd, op_a, b.k(b.i64, 1)); + let b0 = b.bin(b.i64, B::BitAnd, op_b, b.k(b.i64, 1)); + let corr = match a.op { + SimdAluOp::SHAdd | SimdAluOp::UHAdd => { + b.bin(b.i64, B::BitAnd, a0, b0) + } + SimdAluOp::SRHAdd | SimdAluOp::URHAdd => { + b.bin(b.i64, B::BitOr, a0, b0) + } + _ => b.bin( + b.i64, + B::BitAnd, + b.bin(b.i64, B::BitXor, a0, b.k(b.i64, 1)), + b0, + ), + }; + if matches!( + a.op, + SimdAluOp::SHAdd + | SimdAluOp::UHAdd + | SimdAluOp::SRHAdd + | SimdAluOp::URHAdd + ) { + b.bin(b.i64, B::Add, k, corr) + } else { + b.bin(b.i64, B::Sub, k, corr) + } + } + SimdAluOp::SSHl + | SimdAluOp::USHl + | SimdAluOp::SQShl + | SimdAluOp::UQShl + | SimdAluOp::SQRShl + | SimdAluOp::UQRShl => { + let signed_op = matches!( + a.op, + SimdAluOp::SSHl | SimdAluOp::SQShl | SimdAluOp::SQRShl + ); + let rounding = matches!(a.op, SimdAluOp::SQRShl | SimdAluOp::UQRShl); + let saturating = matches!( + a.op, + SimdAluOp::SQShl + | SimdAluOp::UQShl + | SimdAluOp::SQRShl + | SimdAluOp::UQRShl + ); + // Shift amount: the lane's low byte, sign-extended to + // 64 bits (negative means a right shift). + let amt = b.bin( + b.i64, + B::Shl, + b.bin(b.i64, B::BitAnd, bn, b.k(b.i64, 0xff)), + b.k(b.i64, 56), + ); + let amt = b.bin(b.i64, B::Sar, amt, b.k(b.i64, 56)); + let left = b.cmp(C::Sge, amt, b.k(b.i64, 0)); + let n = ssel(left, amt, b.bin(b.i64, B::Sub, b.k(b.i64, 0), amt)); + let shl = |v: Value, k: Value| b.bin(b.i64, B::Shl, v, k); + let sar = |v: Value, k: Value| b.bin(b.i64, B::Sar, v, k); + let shr = |v: Value, k: Value| b.bin(b.i64, B::Shr, v, k); + // Beyond the lane width: a plain right shift keeps the + // sign fill (or zero), a rounding shift collapses to zero, + // and a saturating left shift pins to the lane bound. + let sign_fill = if signed_op { + ssel( + b.cmp(C::Slt, sa, b.k(b.i64, 0)), + b.k(b.i64, emask), + b.k(b.i64, 0), + ) + } else { + b.k(b.i64, 0) + }; + let sat_fill = if signed_op { + ssel( + b.cmp(C::Slt, sa, b.k(b.i64, 0)), + b.k(b.i64, smin), + b.k(b.i64, smax), + ) + } else { + b.k(b.i64, emask) + }; + let far = ssel( + left, + if saturating { sat_fill } else { b.k(b.i64, 0) }, + if rounding { b.k(b.i64, 0) } else { sign_fill }, + ); + // Shift when the amount stays inside the lane width. + // Clamping to `ebits - 1` keeps the host shift defined. + let kc = ssel( + b.cmp(C::Uge, n, b.k(b.i64, ebits)), + b.k(b.i64, ebits - 1), + n, + ); + let rnd = if rounding { + shl(b.k(b.i64, 1), b.bin(b.i64, B::Sub, kc, b.k(b.i64, 1))) + } else { + b.k(b.i64, 0) + }; + let opnd = if signed_op { sa } else { an }; + let t = if rounding { + // The rounding constant must not leak out of the lane. + b.bin( + b.i64, + B::BitAnd, + b.bin(b.i64, B::Add, opnd, rnd), + b.k(b.i64, emask), + ) + } else { + opnd + }; + let right = if rounding { + if signed_op { sar(t, kc) } else { shr(t, kc) } + } else if signed_op { + sar(opnd, kc) + } else { + shr(opnd, kc) + }; + let shifted = shl(opnd, kc); + let moved = ssel(left, shifted, right); + let moved = ssel(b.cmp(C::Uge, n, b.k(b.i64, ebits)), far, moved); + if !saturating { + moved + } else if signed_op { + // Saturation only matters for left shifts; the + // right-shift result already fits the lane. Below + // the 64-bit lane the shifted value is exact, so the + // lane bounds test it directly; a 64-bit lane needs + // the round-trip check because the shift wraps. + let ov = if ebits == 64 { + b.cmp(C::Ne, sar(shifted, kc), opnd) + } else { + b.bin( + b.boolean, + B::BitOr, + b.cmp(C::Sgt, shifted, b.k(b.i64, smax)), + b.cmp(C::Slt, shifted, b.k(b.i64, smin)), + ) + }; + let sat = ssel( + b.cmp(C::Slt, sa, b.k(b.i64, 0)), + b.k(b.i64, smin), + b.k(b.i64, smax), + ); + ssel(left, ssel(ov, sat, moved), moved) + } else { + let ov = if ebits == 64 { + b.cmp(C::Ne, shr(shifted, kc), an) + } else { + b.cmp(C::Ugt, shifted, b.k(b.i64, emask)) + }; + ssel(left, ssel(ov, b.k(b.i64, emask), moved), moved) + } + } + _ => b.k(b.i64, 0), + }; + let masked = b.bin(b.i64, B::BitAnd, r, b.k(b.i64, emask)); + acc = b.bin(b.i64, B::BitOr, acc, b.imm(b.i64, B::Shl, masked, shift)); + } + b.store(at_d + half * 8, acc); + } + } + } +} diff --git a/src/lift/aarch64/src/tests.rs b/src/lift/aarch64/src/tests.rs new file mode 100644 index 0000000..2f88f2a --- /dev/null +++ b/src/lift/aarch64/src/tests.rs @@ -0,0 +1,1006 @@ +//! Tests lift real encodings with a flat test environment and run them on the +//! host through the native JIT. +use crate::{Builder, Environment, LiftError, Lifter, MemoryAccess}; +use alloc::vec::Vec; +use core::mem::offset_of; +use volt_ir::function::{BinOp as B, Function, Value}; +use volt_isa_aarch64::{decode::*, flow::Flow}; +use volt_target::native::{self, JittedFunction}; + +/// Guest state of the test environment. +#[repr(C)] +#[derive(Default)] +struct State { + x: [u64; 31], + sp: u64, + flags: u32, + v: [[u64; 2]; 32], +} + +/// Memory is the host's own: a guest address is a host pointer and every +/// load or store is an ordinary IR load or store. Counts the lifter's hooks. +#[derive(Default)] +struct Flat { + begun: usize, + exits: usize, +} +impl Flat { + /// `address + delta` as a host pointer. + fn host(b: &Builder, address: Value, delta: u64) -> Value { + b.imm(b.ptr, B::Add, address, delta) + } + fn load(b: &Builder, address: Value, delta: u64, size: Size, rt: u8, signed: SignExtend) { + let bits = size as u16 * 8; + let raw = b.load_ptr(b.int(bits), Self::host(b, address, delta)); + let mut v = b.cv(b.i64, raw); + if signed != SignExtend::None && bits < 64 { + let n = 64 - bits as u64; + let extended = b.shift(Width::X64, b.imm(b.i64, B::Shl, v, n), Shift::Asr, n); + v = if signed == SignExtend::To32 { + b.imm(b.i64, B::BitAnd, extended, 0xffff_ffff) + } else { + extended + }; + } + b.put(rt, v); + } + fn store(b: &Builder, address: Value, delta: u64, size: Size, rt: u8) { + let t = b.int(size as u16 * 8); + b.store_ptr( + Self::host(b, address, delta), + b.cv(t, b.reg(b.i64, rt, true)), + ); + } + fn vector(b: &Builder, address: Value, bytes: u8, rt: u8, op: MemoryOp, delta: u64) { + let at = b.v_at(rt); + let chunk = b.int(bytes.min(8) as u16 * 8); + let chunks = if bytes == 16 { 2 } else { 1 }; + for i in 0..chunks { + let host = Self::host(b, address, delta + i as u64 * 8); + if op == MemoryOp::Store { + b.store_ptr(host, b.cv(chunk, b.load(b.i64, at + i * 8))); + } else { + b.store(at + i * 8, b.cv(b.i64, b.load_ptr(chunk, host))); + } + } + if op == MemoryOp::Load && bytes <= 8 { + b.store(at + 8, b.k(b.i64, 0)); + } + } +} +impl Environment for Flat { + fn x_offset(&self, r: u8) -> usize { + offset_of!(State, x) + r as usize * 8 + } + fn sp_offset(&self) -> usize { + offset_of!(State, sp) + } + fn flags_offset(&self) -> usize { + offset_of!(State, flags) + } + fn v_offset(&self, r: u8) -> usize { + offset_of!(State, v) + r as usize * 16 + } + fn begin_instruction(&mut self, _: &Builder) { + self.begun += 1; + } + fn exit(&mut self, b: &Builder, next_pc: Value) { + self.exits += 1; + b.set_ret(next_pc) + } + fn memory(&mut self, b: &Builder, _: u64, access: MemoryAccess<'_>) -> Result<(), LiftError> { + let MemoryAccess { + insn, + address, + writeback, + } = access; + match (*insn, address) { + (Instruction::Memory(a), Some(address)) if a.op == MemoryOp::Store => { + Self::store(b, address, 0, a.size, a.rt) + } + (Instruction::Memory(a), Some(address)) => { + Self::load(b, address, 0, a.size, a.rt, a.signed) + } + (Instruction::Literal(a), Some(address)) => { + Self::load(b, address, 0, a.size, a.rt, a.signed) + } + (Instruction::Pair(a), Some(address)) if a.op == MemoryOp::Store => { + Self::store(b, address, 0, a.size, a.rt); + Self::store(b, address, a.size as u64, a.size, a.rt2); + } + (Instruction::Pair(a), Some(address)) => { + Self::load(b, address, 0, a.size, a.rt, a.signed); + Self::load(b, address, a.size as u64, a.size, a.rt2, a.signed); + } + (Instruction::SimdMemory(a), Some(address)) => { + Self::vector(b, address, a.bytes, a.rt, a.op, 0) + } + (Instruction::SimdLiteral(a), Some(address)) => { + Self::vector(b, address, a.bytes, a.rt, MemoryOp::Load, 0) + } + (Instruction::SimdPair(a), Some(address)) => { + Self::vector(b, address, a.bytes, a.rt, a.op, 0); + Self::vector(b, address, a.bytes, a.rt2, a.op, a.bytes as u64); + } + _ => return Err(LiftError::Unsupported), + } + if let Some(w) = writeback { + b.put_sp(w.register, w.value); + } + Ok(()) + } + fn event(&mut self, _: &Builder, _: u64, _: &Instruction) -> Result<(), LiftError> { + Err(LiftError::Unsupported) + } +} + +/// Lift `bytes` as one block at `pc`: instructions until an architectural branch +/// ends it (plain memory accesses do not), then an exit at the next pc. +fn lift(pc: u64, bytes: &[u8]) -> Result<(Function, Flat), LiftError> { + let mut lifter = Lifter::new(Flat::default()); + let mut at = pc; + let mut ended = false; + for chunk in bytes.chunks_exact(4) { + let word = u32::from_le_bytes(chunk.try_into().unwrap()); + let instruction = decode(word).unwrap(); + ended = instruction.flow(at) != Flow::Next; + lifter.lift(at, instruction)?; + at += 4; + if ended { + break; + } + } + if !ended { + lifter.exit(at); + } + Ok(lifter.into_parts()) +} +struct Block { + _image: JittedFunction, + entry: unsafe extern "C" fn(*mut State) -> u64, +} +impl Block { + fn run(&self, state: &mut State) -> u64 { + // SAFETY: the lifted function has exactly this signature and `_image` + // keeps its mapping alive. + unsafe { (self.entry)(state) } + } +} +fn compile(pc: u64, bytes: &[u8]) -> Result { + let (function, _) = lift(pc, bytes)?; + let image = native::compile_owned(function).unwrap(); + // SAFETY: see `Block::run`. + let entry = unsafe { image.entry() }; + Ok(Block { + _image: image, + entry, + }) +} +fn bytes(words: &[u32]) -> Vec { + words.iter().flat_map(|w| w.to_le_bytes()).collect() +} + +#[test] +fn zero_division_and_signed_overflow_do_not_trap_host() { + // UDIV X2,X0,X1; SDIV X3,X0,X1. + let block = compile(0, &bytes(&[0x9ac10802, 0x9ac10c03])).unwrap(); + let mut cpu = State::default(); + cpu.x[0] = 1 << 63; + cpu.x[1] = 0; + block.run(&mut cpu); + assert_eq!(cpu.x[2], 0); + assert_eq!(cpu.x[3], 0); + cpu.x[1] = u64::MAX; + block.run(&mut cpu); + assert_eq!(cpu.x[3], 1 << 63); +} +#[test] +fn signed_and_unsigned_integer_ops_select_distinct_semantics() { + // sdiv/udiv/asrv (w and x), smulh/umulh, sxtw, sbfx, asr #imm, then + // `cmp` + `cset` for each signed/unsigned condition on w and x. + let words = [ + 0x1ac10c02, 0x1ac10803, 0x1ac12804, 0x9ac12805, 0x9b417c06, 0x9bc17c07, 0x93407c08, + 0x93442c09, 0x13037c0a, 0x6b01001f, 0x1a9fa7eb, 0x1a9fb7ec, 0x1a9f27ed, 0x1a9f97ee, + 0x1a9fd7ef, 0x1a9fc7f0, 0x1a9f87f1, 0xeb01001f, 0x9a9fa7f2, 0x9a9f27f3, 0x9a9fd7f4, + ]; + let block = compile(0, &bytes(&words)).unwrap(); + let pairs: [(u64, u64); 9] = [ + (0xffff_ffff_8000_0005, 3), + (0x8000_0000, 0xffff_ffff), + (0x7fff_ffff_ffff_ffff, 0x8000_0000_0000_0000), + (0x8000_0000_0000_0000, u64::MAX), + (3, 0), + (7, 7), + (0xffff_ffff_0000_0007, 0x0000_0001_0000_0009), + (0x8000_0001, 2), + (2, 0x8000_0001), + ]; + for (a, b) in pairs { + let mut cpu = State::default(); + cpu.x[0] = a; + cpu.x[1] = b; + block.run(&mut cpu); + let (w, wb) = (a as u32, b as u32); + let (sw, swb) = (w as i32, wb as i32); + let sdiv = if wb == 0 { + 0 + } else { + sw.wrapping_div(swb) as u32 + }; + let udiv = if wb == 0 { 0 } else { w / wb }; + assert_eq!(cpu.x[2], sdiv as u64, "sdiv w {a:#x} {b:#x}"); + assert_eq!(cpu.x[3], udiv as u64, "udiv w {a:#x} {b:#x}"); + assert_eq!(cpu.x[4], (sw >> (wb & 31)) as u32 as u64, "asr w"); + assert_eq!(cpu.x[5], ((a as i64) >> (b & 63)) as u64, "asr x"); + assert_eq!( + cpu.x[6], + ((a as i64 as i128 * b as i64 as i128) >> 64) as u64, + "smulh" + ); + assert_eq!(cpu.x[7], ((a as u128 * b as u128) >> 64) as u64, "umulh"); + assert_eq!(cpu.x[8], w as i32 as i64 as u64, "sxtw"); + assert_eq!(cpu.x[9], ((a >> 4) as u8) as i8 as i64 as u64, "sbfx"); + assert_eq!(cpu.x[10], (sw >> 3) as u32 as u64, "asr w imm"); + // Registers 11-17 come from the 32-bit compare, 18-20 from the 64-bit one. + let w_flags = [ + sw < swb, + sw >= swb, + w < wb, + w > wb, + sw > swb, + sw <= swb, + w <= wb, + ]; + for (i, want) in w_flags.into_iter().enumerate() { + assert_eq!(cpu.x[11 + i], want as u64, "w cond {i} {a:#x} {b:#x}"); + } + let (sa, sb) = (a as i64, b as i64); + assert_eq!(cpu.x[18], (sa < sb) as u64, "x lt"); + assert_eq!(cpu.x[19], (a < b) as u64, "x lo"); + assert_eq!(cpu.x[20], (sa > sb) as u64, "x gt"); + } +} +#[test] +fn signed_and_unsigned_vector_lanes_select_distinct_semantics() { + // smax.16b, sabd.8h, shadd.4s, sshl.4s, sqadd.16b, cmgt.2d, cmhi.2d. + let words = [ + 0x4e216403, 0x4e617404, 0x4ea10405, 0x4ea14406, 0x4e210c07, 0x4ee13409, 0x6ee1340a, + ]; + let block = compile(0, &bytes(&words)).unwrap(); + let a: [u8; 16] = [ + 0x80, 0x7f, 0x01, 0xff, 0x10, 0x90, 0x00, 0x7e, 0x85, 0x05, 0xfe, 0x40, 0x7f, 0x80, 0xc0, + 0x20, + ]; + let b: [u8; 16] = [ + 0x01, 0x80, 0xff, 0x01, 0x90, 0x10, 0x00, 0x7f, 0x05, 0x85, 0x02, 0xc0, 0x7f, 0x7f, 0x40, + 0xe0, + ]; + let pack = |x: [u8; 16]| { + let v = u128::from_le_bytes(x); + [v as u64, (v >> 64) as u64] + }; + let mut cpu = State::default(); + cpu.v[0] = pack(a); + cpu.v[1] = pack(b); + block.run(&mut cpu); + let out = |r: usize| (u128::from(cpu.v[r][1]) << 64 | u128::from(cpu.v[r][0])).to_le_bytes(); + let lanes16 = |x: [u8; 16]| -> Vec { + x.chunks(2) + .map(|c| i16::from_le_bytes([c[0], c[1]])) + .collect() + }; + let lanes32 = |x: [u8; 16]| -> Vec { + x.chunks(4) + .map(|c| i32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .collect() + }; + // smax.16b + let want: Vec = (0..16) + .map(|i| (a[i] as i8).max(b[i] as i8) as u8) + .collect(); + assert_eq!(out(3).to_vec(), want, "smax"); + // sabd.8h + let want: Vec = lanes16(a) + .iter() + .zip(lanes16(b)) + .flat_map(|(x, y)| ((*x as i32 - y as i32).unsigned_abs() as u16).to_le_bytes()) + .collect(); + assert_eq!(out(4).to_vec(), want, "sabd"); + // shadd.4s + let want: Vec = lanes32(a) + .iter() + .zip(lanes32(b)) + .flat_map(|(x, y)| (((*x as i64 + y as i64) >> 1) as i32).to_le_bytes()) + .collect(); + assert_eq!(out(5).to_vec(), want, "shadd"); + // sshl.4s: the shift amount is the signed low byte of each lane of b. + let want: Vec = lanes32(a) + .iter() + .zip(lanes32(b)) + .flat_map(|(x, y)| { + let n = y as i8 as i32; + let r = if n >= 32 { + 0 + } else if n >= 0 { + x << n + } else if n > -32 { + x >> -n + } else if *x < 0 { + -1 + } else { + 0 + }; + r.to_le_bytes() + }) + .collect(); + assert_eq!(out(6).to_vec(), want, "sshl"); + // sqadd.16b + let want: Vec = (0..16) + .map(|i| (a[i] as i8).saturating_add(b[i] as i8) as u8) + .collect(); + assert_eq!(out(7).to_vec(), want, "sqadd"); + // cmgt.2d / cmhi.2d + for (r, signed) in [(9, true), (10, false)] { + for lane in 0..2 { + let (x, y) = (cpu.v[0][lane], cpu.v[1][lane]); + let gt = if signed { + (x as i64) > (y as i64) + } else { + x > y + }; + assert_eq!( + cpu.v[r][lane], + if gt { u64::MAX } else { 0 }, + "cmp {r} {lane}" + ); + } + } +} +#[test] +fn simd_compare_zero_masks_each_lane() { + // cmeq v1.16b, v0.16b, #0. + let block = compile(0x1000, &bytes(&[0x4e209801])).unwrap(); + let mut cpu = State::default(); + cpu.v[0][0] = 0x00ff_0000_0000_00ff; + cpu.v[0][1] = 0xffff_ffff_ffff_ffff; + block.run(&mut cpu); + assert_eq!(cpu.v[1][0], 0xff00_ffff_ffff_ff00); + assert_eq!(cpu.v[1][1], 0); + // Half-width form operates on v0.8b and clears the top half. + let block = compile(0x1000, &bytes(&[0x0e209801])).unwrap(); + cpu.v[0][0] = 0x00ff_0000_0000_00ff; + cpu.v[0][1] = 0x1234_5678_9abc_def0; + cpu.v[1] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; + block.run(&mut cpu); + assert_eq!(cpu.v[1][0], 0xff00_ffff_ffff_ff00); + assert_eq!(cpu.v[1][1], 0); +} +#[test] +fn simd_fmov_moves_bits_between_gpr_and_fpr() { + // fmov x2, d2 then fmov d3, x2 round-trips the low doubleword. + let block = compile(0x1000, &bytes(&[0x9e660042, 0x9e670043])).unwrap(); + let mut cpu = State::default(); + cpu.v[2] = [0x1122_3344_5566_7788, 0xdead_beef_dead_beef]; + block.run(&mut cpu); + assert_eq!(cpu.x[2], 0x1122_3344_5566_7788); + assert_eq!(cpu.v[3], [0x1122_3344_5566_7788, 0]); + // 32-bit form truncates to the low word. + let block = compile(0x1000, &bytes(&[0x1e260022])).unwrap(); + cpu.v[1] = [0xaaaa_bbbb_cccc_dddd, 0]; + block.run(&mut cpu); + assert_eq!(cpu.x[2] as u32, 0xcccc_dddd); +} +#[test] +fn simd_pairwise_reductions_match_lane_layout() { + let setup = || { + let mut cpu = State::default(); + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + cpu.v[2] = [0x1112_1314_1516_1718, 0x191a_1b1c_1d1e_1f20]; + cpu + }; + for (word, want) in [ + ( + 0x6e22a420u32, + [0x0a0c_0e10_0204_0608u64, 0x1a1c_1e20_1214_1618], + ), + (0x6e22ac20, [0x090b_0d0f_0103_0507, 0x191b_1d1f_1113_1517]), + (0x4e22bc20, [0x1317_1b1f_0307_0b0f, 0x3337_3b3f_2327_2b2f]), + (0x4ea2bc20, [0x1618_1a1c_0608_0a0c, 0x3638_3a3c_2628_2a2c]), + (0x4ee2bc20, [0x0a0c_0e10_1214_1618, 0x2a2c_2e30_3234_3638]), + ] { + let mut cpu = setup(); + let block = compile(0x1000, &bytes(&[word])).unwrap(); + block.run(&mut cpu); + assert_eq!(cpu.v[0], want, "{word:08x}"); + } +} +#[test] +fn simd_horizontal_add_sums_lanes() { + for (word, lanes, want) in [ + (0x4e31b820u32, 16usize, 0x88u64), + (0x4e71b820, 8, 0x4048), + (0x4eb1b820, 4, 0x1c20_2428), + ] { + let block = compile(0x1000, &bytes(&[word])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + block.run(&mut cpu); + let _ = lanes; + assert_eq!(cpu.v[0], [want, 0], "{word:08x}"); + } + // The 64-bit form reads only the low half of the source. + let block = compile(0x1000, &bytes(&[0x0e31b820])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + block.run(&mut cpu); + assert_eq!(cpu.v[0], [0x24, 0]); +} +#[test] +fn simd_dup_from_element_replicates_a_lane() { + for (word, want) in [ + // dup v0.4h, v1.h[0] -> element 0x0708 in every halfword. + (0x0e020420u32, [0x0708_0708_0708_0708u64, 0]), + // dup v0.8h, v1.h[7] -> element 0x090a, both halves. + (0x4e1e0420, [0x090a_090a_090a_090a, 0x090a_090a_090a_090a]), + // dup v0.4s, v1.s[1] -> element 0x0102_0304. + (0x4e0c0420, [0x0102_0304_0102_0304, 0x0102_0304_0102_0304]), + // dup v0.2d, v1.d[1] -> element 0x090a_0b0c_0d0e_0f10. + (0x4e180420, [0x090a_0b0c_0d0e_0f10, 0x090a_0b0c_0d0e_0f10]), + // dup v0.16b, v1.b[3] -> element 0x05. + (0x4e070420, [0x0505_0505_0505_0505, 0x0505_0505_0505_0505]), + ] { + let mut cpu = State::default(); + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + let block = compile(0x1000, &bytes(&[word])).unwrap(); + block.run(&mut cpu); + assert_eq!(cpu.v[0], want, "{word:08x}"); + } +} +#[test] +fn simd_ext_uses_the_arrangement_it_encodes() { + let setup = || { + let mut cpu = State::default(); + cpu.v[0] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; + cpu.v[1] = [0x0102_0304_0506_0708, 0x090a_0b0c_0d0e_0f10]; + cpu.v[2] = [0x1112_1314_1516_1718, 0x191a_1b1c_1d1e_1f20]; + cpu + }; + // ext v0.8b, v1.8b, v2.8b, #3 windows the low halves and clears the upper half. + let mut cpu = setup(); + compile(0x1000, &bytes(&[0x2e021820])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.v[0], [0x1617_1801_0203_0405, 0]); + // ext v0.16b, v1.16b, v2.16b, #3. + let mut cpu = setup(); + compile(0x1000, &bytes(&[0x6e021820])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.v[0], [0x0e0f_1001_0203_0405, 0x1617_1809_0a0b_0c0d]); +} +#[test] +fn simd_modified_immediates_write_the_expanded_pattern() { + for (word, want) in [ + // movi v0.2d, #0 and movi d0, #0xff (which clears the upper half). + (0x6f00e400u32, [0u64, 0]), + (0x2f00e420, [0xff, 0]), + // mvni v31.4s, #0xff inverts every 32-bit lane. + (0x6f0707ff, [0xffff_ff00_ffff_ff00; 2]), + // fmov v0.8h/.4h, #2.0 and #-2.5 expand the half-precision pattern. + (0x4f00fc00, [0x4000_4000_4000_4000; 2]), + (0x0f00fc00, [0x4000_4000_4000_4000, 0]), + (0x4f04fc80, [0xc100_c100_c100_c100; 2]), + ] { + let rd = (word & 31) as usize; + let mut cpu = State::default(); + cpu.v[rd] = [0x5555_5555_5555_5555; 2]; + compile(0x1000, &bytes(&[word])).unwrap().run(&mut cpu); + assert_eq!(cpu.v[rd], want, "{word:08x}"); + } +} +#[test] +fn simd_fmov_upper_doubleword_keeps_the_lower_half() { + let mut cpu = State::default(); + cpu.v[0] = [0x1111_1111_1111_1111, 0x2222_2222_2222_2222]; + cpu.x[3] = 0x3333_3333_3333_3333; + // fmov v0.d[1], x3. + compile(0x1000, &bytes(&[0x9eaf0060])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.v[0], [0x1111_1111_1111_1111, 0x3333_3333_3333_3333]); + // fmov x4, v0.d[1]. + cpu.v[0][1] = 0x4444_4444_4444_4444; + compile(0x1000, &bytes(&[0x9eae0004])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.x[4], 0x4444_4444_4444_4444); +} +#[test] +fn simd_table_lookup_selects_bytes() { + // tbl v0.16b, {v1.16b, v2.16b}, v3.16b with indices 0,2,4,... + let block = compile(0x1000, &bytes(&[0x4e032020])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; + cpu.v[2] = [0x1716_1514_1312_1110, 0x1f1e_1d1c_1b1a_1918]; + cpu.v[3] = [0x0604_0200_0e0c_0a08, 0x1614_1210_1e1c_1a18]; + block.run(&mut cpu); + assert_eq!(cpu.v[0], [0x0604_0200_0e0c_0a08, 0x1614_1210_1e1c_1a18]); + // Out-of-range indices yield zero for tbl. + let block = compile(0x1000, &bytes(&[0x4e020020])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; + cpu.v[2] = [0xff01_ff00_ff00_ff00, 0xffff_ffff_ffff_ffff]; + block.run(&mut cpu); + assert_eq!(cpu.v[0], [0x0001_0000_0000_0000, 0]); + // tbx keeps the destination byte where the index is out of range. + let block = compile(0x1000, &bytes(&[0x4e021020])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = [0x0706_0504_0302_0100, 0x0f0e_0d0c_0b0a_0908]; + cpu.v[2] = [0xff01_ff00_ff00_ff00, 0xffff_ffff_ffff_ffff]; + cpu.v[0] = [0xdead_beef_dead_beef, 0xdead_beef_dead_beef]; + block.run(&mut cpu); + assert_eq!(cpu.v[0], [0xde01_be00_de00_be00, 0xdead_beef_dead_beef]); +} +#[test] +fn simd_alu_lane_semantics() { + // Ground truth computed with the host's own byte arithmetic. + struct Case { + word: u32, + a: [u64; 2], + b: [u64; 2], + want: [u64; 2], + } + let cases = [ + // add v0.16b, v1.16b, v2.16b wraps per byte + Case { + word: 0x4e228420, + a: [0x0102_03ff_0405_0607, 0x0f10_1120_2122_2324], + b: [0x0101_0101_0101_0101, 0x0102_0304_0506_0708], + want: [0x0203_0400_0506_0708, 0x1012_1424_2628_2a2c], + }, + // sub v0.16b, v1.16b, v2.16b + Case { + word: 0x6e228420, + a: [0x0001_0203_0405_0607, 0], + b: [0x0101_0101_0101_0101, 0], + want: [0xff00_0102_0304_0506, 0], + }, + // cmhs v0.16b, v1.16b, v2.16b (unsigned >=) + Case { + word: 0x6e223c20, + a: [0x00ff_0080_7f7f_8000, 0], + b: [0x0001_0180_7f80_8000, 0], + want: [0xffff_00ff_ff00_ffff, 0xffff_ffff_ffff_ffff], + }, + // umin v0.8h, v1.8h, v2.8h + Case { + word: 0x6e626c20, + a: [0x0003_0002_0001_0000, 0], + b: [0x0002_0003_0000_0001, 0], + want: [0x0002_0002_0000_0000, 0], + }, + // uqadd v0.8h, v1.8h, v2.8h saturates + Case { + word: 0x6e620c20, + a: [0xffff_8000_0001_0000, 0], + b: [0x0001_9000_0002_0001, 0], + want: [0xffff_ffff_0003_0001, 0], + }, + // shadd v0.16b, v1.16b, v2.16b rounds down (arithmetic) + Case { + word: 0x4e220420, + a: [0x0303_fefe_0505_00ff, 0], + b: [0x0303_fefe_0505_00ff, 0], + want: [0x0303_fefe_0505_00ff, 0], + }, + // sshl v0.8h by signed amounts in the low byte of each lane + Case { + word: 0x4e624420, + a: [0x0004_0003_0002_0001, 0], + b: [0x0000_0002_00ff_0001, 0], + want: [0x0004_000c_0001_0002, 0], + }, + // mla/mls fold in the destination *lane*, not the whole half + Case { + word: 0x4e229420, + a: [0x0202_0202_0202_0202, 0], + b: [0x0505_0505_0505_0505, 0], + want: [0x0a0a_0a0a_0a0a_0a0a, 0], + }, + Case { + word: 0x6e229420, + a: [0x0202_0202_0202_0202, 0], + b: [0x0505_0505_0505_0505, 0], + want: [0xf6f6_f6f6_f6f6_f6f6, 0], + }, + // bsl: destination starts zero, so the result is v2 unchanged + Case { + word: 0x6e621c20, + a: [0xaaaa_aaaa_aaaa_aaaa, 0], + b: [0x5555_5555_5555_5555, 0], + want: [0x5555_5555_5555_5555, 0], + }, + // bit with a zero destination inserts the masked v1 bits + Case { + word: 0x6ea21c20, + a: [0x0f0f_0f0f_0f0f_0f0f, 0], + b: [0xaaaa_aaaa_aaaa_aaaa, 0], + want: [0x0a0a_0a0a_0a0a_0a0a, 0], + }, + // bif with a zero destination inserts the inverse-masked bits + Case { + word: 0x6ee21c20, + a: [0x0f0f_0f0f_0f0f_0f0f, 0], + b: [0xaaaa_aaaa_aaaa_aaaa, 0], + want: [0x0505_0505_0505_0505, 0], + }, + // shadd of two -1 bytes stays -1 (arithmetic halving) + Case { + word: 0x4e220420, + a: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], + b: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], + want: [0xffff_ffff_ffff_ffff, 0xffff_ffff_ffff_ffff], + }, + // shsub of 0 - 1 halves to -1 + Case { + word: 0x4e222420, + a: [0, 0], + b: [0x0101_0101_0101_0101, 0], + want: [0xffff_ffff_ffff_ffff, 0], + }, + // srhadd of 1 and 2 rounds to 2 + Case { + word: 0x4e221420, + a: [0x0101_0101_0101_0101, 0], + b: [0x0202_0202_0202_0202, 0], + want: [0x0202_0202_0202_0202, 0], + }, + // sshl v0.8h: lane 0 shifts left by 1, lane 1 right by 1 + Case { + word: 0x4e624420, + a: [0x0008_0004_0002_0001, 0], + b: [0x0000_00ff_0001_0001, 0], + want: [0x0008_0002_0004_0002, 0], + }, + // uqshl v0.8h saturates an overflow to the lane maximum + Case { + word: 0x6e624c20, + a: [0x0000_0000_0000_8000, 0], + b: [0x0000_0000_0000_0001, 0], + want: [0x0000_0000_0000_ffff, 0], + }, + // sqshl v0.8h saturates a positive overflow to the lane maximum + Case { + word: 0x4e624c20, + a: [0x0000_0000_0000_4000, 0], + b: [0x0000_0000_0000_0001, 0], + want: [0x0000_0000_0000_7fff, 0], + }, + // sqrshl v0.8h rounds before shifting right + Case { + word: 0x4e625c20, + a: [0x0000_0000_0000_0003, 0], + b: [0x0000_0000_0000_00ff, 0], + want: [0x0000_0000_0000_0002, 0], + }, + ]; + for case in cases { + let block = compile(0x1000, &bytes(&[case.word])).unwrap(); + let mut cpu = State::default(); + cpu.v[1] = case.a; + cpu.v[2] = case.b; + block.run(&mut cpu); + assert_eq!(cpu.v[0], case.want, "{:08x}", case.word); + } +} + +fn sub_flags(a: u64, b: u64) -> u32 { + let r = a.wrapping_sub(b); + let (n, z, c) = (r >> 63, r == 0, a >= b); + let v = ((a ^ b) & (a ^ r)) >> 63; + (n as u32) << 31 | (z as u32) << 30 | (c as u32) << 29 | (v as u32) << 28 +} +fn add_flags32(a: u32, b: u32) -> u32 { + let (r, c) = a.overflowing_add(b); + let v = (!(a ^ b) & (a ^ r)) >> 31; + (r >> 31) << 31 | ((r == 0) as u32) << 30 | (c as u32) << 29 | v << 28 +} +#[test] +fn subs_and_adds_set_nzcv() { + // subs x2,x0,x1; adds w3,w0,w1. + let block = compile(0, &bytes(&[0xeb010002, 0x2b010003])).unwrap(); + for (a, b) in [ + (5u64, 5u64), + (3, 5), + (5, 3), + (1 << 63, 1), + (0xffff_ffff, 1), + (0x7fff_ffff, 1), + (u64::MAX, u64::MAX), + ] { + let mut cpu = State::default(); + cpu.x[0] = a; + cpu.x[1] = b; + block.run(&mut cpu); + assert_eq!(cpu.x[2], a.wrapping_sub(b)); + assert_eq!(cpu.x[3], (a as u32).wrapping_add(b as u32) as u64); + assert_eq!( + cpu.flags, + add_flags32(a as u32, b as u32), + "adds {a:#x} {b:#x}" + ); + // The flags of the first instruction were overwritten by the second; + // check them on their own. + let sub = compile(0, &bytes(&[0xeb010002])).unwrap(); + sub.run(&mut cpu); + assert_eq!(cpu.flags, sub_flags(a, b), "subs {a:#x} {b:#x}"); + } +} +#[test] +fn conditional_select_and_compare_read_the_flags() { + // subs x2,x0,x1; csel x4,x0,x1,lt. + let block = compile(0, &bytes(&[0xeb010002, 0x9a81b004])).unwrap(); + for (a, b) in [(3u64, 5u64), (5, 3), (5, 5), (1 << 63, 1)] { + let mut cpu = State::default(); + cpu.x[0] = a; + cpu.x[1] = b; + block.run(&mut cpu); + let want = if (a as i64) < (b as i64) { a } else { b }; + assert_eq!(cpu.x[4], want, "csel lt {a:#x} {b:#x}"); + } + // ccmp x0,x1,#4,eq: compares when EQ holds, otherwise loads the immediate. + let block = compile(0, &bytes(&[0xfa410004])).unwrap(); + let mut cpu = State::default(); + cpu.x[0] = 7; + cpu.x[1] = 9; + cpu.flags = 0x4000_0000; + block.run(&mut cpu); + assert_eq!(cpu.flags, sub_flags(7, 9)); + cpu.flags = 0; + block.run(&mut cpu); + assert_eq!(cpu.flags, 0x4000_0000); +} +#[test] +fn zero_register_destinations_do_not_overwrite_sp() { + // ORR XZR,X0,X0; ADR XZR,. ; ADDS XZR,X0,X0. + let block = compile(0x1000, &bytes(&[0xaa00001f, 0x1000001f, 0xab00001f])).unwrap(); + let mut cpu = State::default(); + cpu.sp = 0x9000; + cpu.x[0] = 0x1234; + block.run(&mut cpu); + assert_eq!(cpu.sp, 0x9000); + // CBZ XZR,+8 must take the branch independently of SP. + let branch = compile(0x2000, &bytes(&[0xb400005f])).unwrap(); + assert_eq!(branch.run(&mut cpu), 0x2008); +} +#[test] +fn branches_return_the_next_pc() { + let mut cpu = State::default(); + cpu.x[0] = 1; + cpu.x[5] = 0xdead_0000; + // b +8; bl +16; br x5; cbnz x0,+8 (taken) and with x0 == 0 (not taken). + assert_eq!( + compile(0x1000, &bytes(&[0x14000002])) + .unwrap() + .run(&mut cpu), + 0x1008 + ); + assert_eq!( + compile(0x1000, &bytes(&[0x94000004])) + .unwrap() + .run(&mut cpu), + 0x1010 + ); + assert_eq!(cpu.x[30], 0x1004); + assert_eq!( + compile(0x1000, &bytes(&[0xd61f00a0])) + .unwrap() + .run(&mut cpu), + 0xdead_0000 + ); + let cbnz = compile(0x1000, &bytes(&[0xb5000040])).unwrap(); + assert_eq!(cbnz.run(&mut cpu), 0x1008); + cpu.x[0] = 0; + assert_eq!(cbnz.run(&mut cpu), 0x1004); +} +#[test] +fn the_environment_sees_every_instruction_and_every_exit() { + // add x0,x0,#1 twice, then fall off the end: two instructions, one exit. + let (_, env) = lift(0x1000, &bytes(&[0x91000400, 0x91000400])).unwrap(); + assert_eq!((env.begun, env.exits), (2, 1)); + // A conditional branch exits once, for both outcomes. + let (_, env) = lift(0x1000, &bytes(&[0xb5000040, 0x91000400])).unwrap(); + assert_eq!((env.begun, env.exits), (1, 1)); +} +#[test] +fn exclusives_and_system_instructions_are_the_environments_business() { + // ldxr x1,[x0]; mrs x1,nzcv. + for word in [0xc85f7c01u32, 0xd53b4201] { + assert_eq!( + lift(0, &bytes(&[word])).err(), + Some(LiftError::Unsupported), + "{word:08x}" + ); + } +} +#[test] +fn plain_loads_and_stores_reach_the_environment_with_their_address() { + let mut buf = [0u64; 8]; + let base = buf.as_mut_ptr() as u64; + let mut cpu = State::default(); + cpu.x[0] = base; + cpu.x[1] = 0x1122_3344_5566_7788; + // str x1,[x0,#16]; ldr x3,[x0,#16]; ldr x4,[x0,x2,lsl #3] with x2 = 2. + cpu.x[2] = 2; + compile(0, &bytes(&[0xf9000801, 0xf9400803, 0xf8627804])) + .unwrap() + .run(&mut cpu); + assert_eq!(buf[2], 0x1122_3344_5566_7788); + assert_eq!(cpu.x[3], 0x1122_3344_5566_7788); + assert_eq!(cpu.x[4], 0x1122_3344_5566_7788); +} +#[test] +fn narrow_loads_extend_as_the_instruction_says() { + let mut buf = [0u64; 1]; + buf[0] = 0xffff_ffff_8765_f0f1; + let mut cpu = State::default(); + for (word, want) in [ + (0x39400001u32, 0xf1u64), // ldrb w1 + (0x39800001, 0xffff_ffff_ffff_fff1), // ldrsb x1 + (0xb9800001, 0xffff_ffff_8765_f0f1), // ldrsw x1 + ] { + cpu.x[0] = buf.as_mut_ptr() as u64; + compile(0, &bytes(&[word])).unwrap().run(&mut cpu); + assert_eq!(cpu.x[1], want, "{word:08x}"); + } + // strh w1,[x0,#2] stores only the low halfword. + cpu.x[0] = buf.as_mut_ptr() as u64; + cpu.x[1] = 0xaaaa_bbbb_cccc_dddd; + compile(0, &bytes(&[0x79000401])).unwrap().run(&mut cpu); + assert_eq!(buf[0], 0xffff_ffff_dddd_f0f1); +} +#[test] +fn pair_accesses_apply_the_writeback_after_the_access() { + let mut buf = [0x11u64, 0x22, 0x33, 0x44]; + let mut cpu = State::default(); + let base = buf.as_mut_ptr() as u64; + // ldp x1,x2,[x0],#16: loads from the old base, then advances it. + cpu.x[0] = base; + compile(0, &bytes(&[0xa8c10801])).unwrap().run(&mut cpu); + assert_eq!((cpu.x[1], cpu.x[2], cpu.x[0]), (0x11, 0x22, base + 16)); + // ldp x1,x2,[x0,#16]!: the access uses the updated base. + cpu.x[0] = base - 16; + compile(0, &bytes(&[0xa9c10801])).unwrap().run(&mut cpu); + assert_eq!((cpu.x[1], cpu.x[2], cpu.x[0]), (0x11, 0x22, base)); + // stp x1,x2,[x0] + cpu.x[0] = base + 16; + cpu.x[1] = 0xaa; + cpu.x[2] = 0xbb; + compile(0, &bytes(&[0xa9000801])).unwrap().run(&mut cpu); + assert_eq!((buf[2], buf[3]), (0xaa, 0xbb)); +} +#[test] +fn literal_loads_use_the_pc_relative_address() { + let buf = [0x0123_4567_89ab_cdefu64, 0xfedc_ba98_7654_3210]; + let mut cpu = State::default(); + // ldr x1,.+8 at a pc that makes the target buf[1]: pc = &buf[0]. + let pc = buf.as_ptr() as u64; + compile(pc, &bytes(&[0x58000041])).unwrap().run(&mut cpu); + assert_eq!(cpu.x[1], 0xfedc_ba98_7654_3210); +} +#[test] +fn vector_loads_and_stores_move_whole_registers() { + let mut buf = [0x1111_2222_3333_4444u64, 0x5555_6666_7777_8888, 0, 0, 0, 0]; + let mut cpu = State::default(); + cpu.x[0] = buf.as_mut_ptr() as u64; + cpu.v[1] = [u64::MAX, u64::MAX]; + // ldr q1,[x0]; str q1,[x0,#16]; ldr d2,[x0,#8] zeroes the top half. + cpu.v[2] = [u64::MAX, u64::MAX]; + compile(0, &bytes(&[0x3dc00001, 0x3d800401, 0xfd400402])) + .unwrap() + .run(&mut cpu); + assert_eq!(cpu.v[1], [0x1111_2222_3333_4444, 0x5555_6666_7777_8888]); + assert_eq!( + (buf[2], buf[3]), + (0x1111_2222_3333_4444, 0x5555_6666_7777_8888) + ); + assert_eq!(cpu.v[2], [0x5555_6666_7777_8888, 0]); + // ldp q1,q2,[x0] loads two registers: the original pair, and the copy stored above. + cpu.v[1] = [0; 2]; + cpu.v[2] = [0; 2]; + compile(0, &bytes(&[0xad400801])).unwrap().run(&mut cpu); + assert_eq!(cpu.v[1], [0x1111_2222_3333_4444, 0x5555_6666_7777_8888]); + assert_eq!(cpu.v[2], [0x1111_2222_3333_4444, 0x5555_6666_7777_8888]); +} +#[test] +fn simd_pairwise_ops_match_a_reference_model() { + // ADDP/SMAXP/SMINP/UMAXP/UMINP, every element size and both vector widths, + // against random operands. Lane values are compared as the instruction + // defines them: signed forms order the lanes as signed numbers. + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + // (base word, name): Q=0, U=0, size=0, Rm=0, Rn=0, Rd=0. + #[derive(Clone, Copy, PartialEq)] + enum Op { + Add, + SMax, + SMin, + UMax, + UMin, + } + let ops = [Op::Add, Op::SMax, Op::SMin, Op::UMax, Op::UMin]; + for op in ops { + for size in 0..4u32 { + if size == 3 && op != Op::Add { + continue; + } + for q in [false, true] { + if size == 3 && !q { + continue; // ADDP .1D is reserved + } + let (opcode, u) = match op { + Op::Add => (0b10111, 0), + Op::SMax => (0b10100, 0), + Op::SMin => (0b10101, 0), + Op::UMax => (0b10100, 1), + Op::UMin => (0b10101, 1), + }; + let word = 0x0e20_0400u32 + | (q as u32) << 30 + | u << 29 + | size << 22 + | 2 << 16 + | opcode << 11 + | 1 << 5; + let block = compile(0, &bytes(&[word])).unwrap(); + let esize = 8usize << size; + let lanes = (if q { 128 } else { 64 }) / esize; + let mask = if esize == 64 { + u64::MAX + } else { + (1u64 << esize) - 1 + }; + for _ in 0..200 { + let mut cpu = State::default(); + cpu.v[1] = [next(), next()]; + cpu.v[2] = [next(), next()]; + cpu.v[0] = [next(), next()]; // must be fully overwritten + let lane = |v: [u64; 2], i: usize| -> u64 { + let bit = i * esize; + (v[bit / 64] >> (bit % 64)) & mask + }; + let sext = |x: u64| -> i64 { ((x << (64 - esize)) as i64) >> (64 - esize) }; + // Vm:Vn concatenated, `lanes` lanes each. + let all: Vec = (0..lanes) + .map(|i| lane(cpu.v[1], i)) + .chain((0..lanes).map(|i| lane(cpu.v[2], i))) + .collect(); + let mut want = [0u64; 2]; + for i in 0..lanes { + let (a, b) = (all[2 * i], all[2 * i + 1]); + let r = match op { + Op::Add => a.wrapping_add(b) & mask, + Op::SMax => sext(a).max(sext(b)) as u64 & mask, + Op::SMin => sext(a).min(sext(b)) as u64 & mask, + Op::UMax => a.max(b), + Op::UMin => a.min(b), + }; + let bit = i * esize; + want[bit / 64] |= r << (bit % 64); + } + let input = [cpu.v[1], cpu.v[2]]; + block.run(&mut cpu); + assert_eq!( + cpu.v[0], want, + "{word:08x} size {size} q {q} inputs {input:x?}" + ); + } + } + } + } +} diff --git a/src/target/Cargo.toml b/src/target/Cargo.toml index e41cbec..d50e7bf 100644 --- a/src/target/Cargo.toml +++ b/src/target/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [dependencies] object.workspace = true volt-ir.workspace = true +volt-isa-aarch64.workspace = true libc.workspace = true [lints] diff --git a/src/target/src/aarch64/disasm.rs b/src/target/src/aarch64/disasm.rs deleted file mode 100644 index 0d2247e..0000000 --- a/src/target/src/aarch64/disasm.rs +++ /dev/null @@ -1,1581 +0,0 @@ -//! AArch64 (A64) disassembler: the inverse of [`super::encode`]. -//! -//! It decodes exactly the instructions the encoder emits into a readable -//! listing, so codegen debugging does not mean hand-decoding hex. Field -//! positions mirror the encoder bit-for-bit. Anything outside the emitted -//! subset prints as `.word 0x` so nothing is silently mis-decoded. -//! -//! [`one`] renders a single 32-bit instruction word; [`format`] renders a -//! whole code buffer with byte addresses. -//! -//! This module is intentionally self-contained (it operates on raw words and -//! shares no types with the encoder) so it keeps compiling while sibling -//! modules evolve. - -use alloc::string::{String, ToString as _}; -use core::fmt::Write as _; - -/// A source-line-table row: the byte offset where a source line's code begins. -pub struct SourceLine { - pub offset: u32, - pub line: u32, -} - -/// A function's location in a linked image: its name and its word offset. -pub struct Sym<'a> { - pub name: &'a str, - pub word: usize, -} - -/// A defined symbol by absolute address (a function or data object), for -/// resolving branch and `adrp` targets when disassembling an ELF image. -pub struct AddrSym<'a> { - pub name: &'a str, - pub addr: u64, - pub size: u64, - pub is_func: bool, -} - -/// A decoded source-line row keyed by absolute address (from a `.debug_line` -/// program), for objdump `-S`-style annotation of a real ELF. -pub struct AddrLine { - pub addr: u64, - pub line: u32, -} - -/// The context that makes a listing absolute: the current instruction's -/// address and the image's symbols. When present, PC-relative targets render -/// as `0x ` (objdump style). -#[derive(Clone, Copy)] -struct Context<'a> { - pc: u64, - syms: &'a [AddrSym<'a>], -} - -/// Render one instruction word as `mnemonic operands`. -pub fn one(word: u32) -> String { - let mut out = String::new(); - decode(&mut out, word, None); - out -} - -/// Render a whole instruction stream (one word per line, `addr: hex text`). -pub fn format(code: &[u32]) -> String { - let mut out = String::new(); - for (i, &word) in code.iter().enumerate() { - let _ = write!(out, "{:04x}: {:08x} ", i * 4, word); - decode(&mut out, word, None); - out.push('\n'); - } - out -} - -/// Render a listing with source-line markers interleaved (objdump `-S` -/// style): a `; line N` header precedes the instructions belonging to that -/// source line. `lines` must be sorted by offset (as isel produces). -pub fn format_with_lines(code: &[u32], lines: &[SourceLine]) -> String { - let mut out = String::new(); - let mut li: usize = 0; - for (i, &word) in code.iter().enumerate() { - let byte_off: u32 = (i * 4) as u32; - while li < lines.len() && lines[li].offset == byte_off { - let _ = writeln!(out, "; line {}", lines[li].line); - li += 1; - } - let _ = write!(out, "{byte_off:04x}: {word:08x} "); - decode(&mut out, word, None); - out.push('\n'); - } - out -} - -/// Render a whole linked module: each function gets a `name:` label at its -/// offset, and every resolved `bl` is annotated with the callee's name -/// (``), so a linked image reads like a symbolized listing. -pub fn format_module(code: &[u32], syms: &[Sym<'_>]) -> String { - let mut out = String::new(); - for (i, &word) in code.iter().enumerate() { - for s in syms { - if s.word == i { - if i != 0 { - out.push('\n'); - } - let _ = writeln!(out, "{}:", s.name); - } - } - let _ = write!(out, "{:04x}: {:08x} ", i * 4, word); - decode(&mut out, word, None); - // Annotate any PC-relative branch (call or local) with the function - // it lands in, as `` or ``, the way objdump does. - if let Some(target) = branch_target_word(word, i) - && let Some(s) = containing_sym(syms, target) - { - let off = target - s.word as i64; - if off == 0 { - let _ = write!(out, " <{}>", s.name); - } else { - let _ = write!(out, " <{}+0x{:x}>", s.name, off as u64 * 4); - } - } - out.push('\n'); - } - out -} - -/// Render an ELF code section as an objdump-style listing: absolute -/// addresses, function labels, and branch/`adrp` targets resolved to -/// `0x `. `base` is the section's load address; `syms` are all -/// defined symbols (functions and data). -pub fn format_elf(code: &[u32], base: u64, syms: &[AddrSym<'_>]) -> String { - format_elf_impl(code, base, syms, &[]) -} - -/// Like [`format_elf`], but interleaves `; line N` markers from `lines` -/// (sorted by address) before the instruction at each address, so a listing -/// of a `-g` object reads with its source lines. -pub fn format_elf_with_lines( - code: &[u32], - base: u64, - syms: &[AddrSym<'_>], - lines: &[AddrLine], -) -> String { - format_elf_impl(code, base, syms, lines) -} - -fn format_elf_impl(code: &[u32], base: u64, syms: &[AddrSym<'_>], lines: &[AddrLine]) -> String { - let mut out = String::new(); - let mut li: usize = 0; - for (i, &word) in code.iter().enumerate() { - let addr = base + i as u64 * 4; - for s in syms { - if s.is_func && s.addr == addr { - if i != 0 { - out.push('\n'); - } - let _ = writeln!(out, "{addr:016x} <{}>:", s.name); - } - } - while li < lines.len() && lines[li].addr < addr { - li += 1; // skip any before this insn - } - while li < lines.len() && lines[li].addr == addr { - let _ = writeln!(out, "; line {}", lines[li].line); - li += 1; - } - let _ = write!(out, "{addr:08x}: {word:08x} "); - decode(&mut out, word, Some(Context { pc: addr, syms })); - out.push('\n'); - } - out -} - -/// Render a PC-relative code target `rel_bytes` from the current PC: -/// absolute `0x ` when a context is present, else the -/// self-contained `.±N` form. -fn render_target(out: &mut String, ctx: Option>, rel_bytes: i64) { - if let Some(c) = ctx { - // Wrapping add (like the adrp path): a disassembler must never panic - // on the bytes it is given, including a branch whose displacement - // runs below zero. - let abs: u64 = (c.pc as i64).wrapping_add(rel_bytes) as u64; - let _ = write!(out, "0x{abs:x}"); - annotate_sym(out, c.syms, abs); - } else { - out.push('.'); - out.push(if rel_bytes < 0 { '-' } else { '+' }); - out.push_str(&rel_bytes.unsigned_abs().to_string()); - } -} - -/// Append ` ` for the symbol that contains `addr`, if any. -fn annotate_sym(out: &mut String, syms: &[AddrSym<'_>], addr: u64) { - if let Some((name, off)) = symbol_at(syms, addr) { - if off == 0 { - let _ = write!(out, " <{name}>"); - } else { - let _ = write!(out, " <{name}+0x{off:x}>"); - } - } -} - -/// The symbol containing `addr` (the greatest-addressed symbol at or before -/// it): its name and the byte offset into it. -fn symbol_at<'a>(syms: &'a [AddrSym<'a>], addr: u64) -> Option<(&'a str, u64)> { - let mut best: Option<&AddrSym<'_>> = None; - for s in syms { - if s.addr <= addr && best.is_none_or(|b| s.addr > b.addr) { - best = Some(s); - } - } - best.map(|s| (s.name, addr - s.addr)) -} - -/// The target word index of a PC-relative branch (b/bl/b.cond/cbz/cbnz/ -/// tbz/tbnz) at `index`, or `None` if `w` is not such a branch. -fn branch_target_word(w: u32, index: usize) -> Option { - let base: i64 = index as i64; - if w & 0x7C00_0000 == 0x1400_0000 { - // b / bl: imm26 - return Some(base + (((w << 6) as i32) >> 6) as i64); - } - if w & 0xFF00_0010 == 0x5400_0000 || w & 0x7E00_0000 == 0x3400_0000 { - // b.cond / cbz / cbnz: imm19 - return Some(base + (((w << 8) as i32) >> 13) as i64); - } - if w & 0x7E00_0000 == 0x3600_0000 { - // tbz / tbnz: imm14 - return Some(base + (((w << 13) as i32) >> 18) as i64); - } - None -} - -/// The symbol whose range contains word `target` (the greatest symbol at or -/// before it), or `None`. -fn containing_sym<'a>(syms: &'a [Sym<'a>], target: i64) -> Option<&'a Sym<'a>> { - if target < 0 { - return None; - } - let mut best: Option<&'a Sym<'a>> = None; - for s in syms { - if s.word as i64 <= target && best.is_none_or(|b| s.word > b.word) { - best = Some(s); - } - } - best -} - -fn rd(w: u32) -> u32 { - w & 0x1F -} -fn rn(w: u32) -> u32 { - (w >> 5) & 0x1F -} -fn rm(w: u32) -> u32 { - (w >> 16) & 0x1F -} -fn ra(w: u32) -> u32 { - (w >> 10) & 0x1F -} -fn sf(w: u32) -> bool { - (w >> 31) & 1 != 0 -} - -/// A 32/64-bit general register (`w`/`x`), rendering 31 as the zero register. -fn gp(out: &mut String, x64: bool, r: u32) { - let c: char = if x64 { 'x' } else { 'w' }; - if r == 31 { - let _ = write!(out, "{c}zr"); - } else { - let _ = write!(out, "{c}{r}"); - } -} - -/// A general register in an address/stack context, rendering 31 as `sp`. -fn sp(out: &mut String, x64: bool, r: u32) { - if r == 31 { - out.push_str("sp"); - } else { - gp(out, x64, r); - } -} - -fn cond_name(v: u32) -> &'static str { - // binutils spelling (cs/cc rather than the hs/lo synonyms) so listings - // match objdump. - match v { - 0 => "eq", - 1 => "ne", - 2 => "cs", - 3 => "cc", - 4 => "mi", - 5 => "pl", - 6 => "vs", - 7 => "vc", - 8 => "hi", - 9 => "ls", - 10 => "ge", - 11 => "lt", - 12 => "gt", - 13 => "le", - 14 => "al", - 15 => "nv", - _ => "??", - } -} - -/// Masks for the three-register data-processing form: clear rd, rn, rm; keep -/// the opcode and the op2/shift field in bits[10:15]. The `sf` bit is dropped -/// so one entry covers both the w and x forms. -const RRR_MASK: u32 = 0x7FE0_FC00; - -/// `RRR_MASK` with the imm6 shift-amount field (bits[15:10]) dropped, for the -/// add/sub shifted-register form where that field is a real operand and not -/// part of the opcode. -const RRR_NO_IMM6_MASK: u32 = RRR_MASK & !0xFC00u32; - -/// The 32-bit-base opcodes of the shared three-register form (sf dropped). -const RRR_TABLE: &[(u32, &str)] = &[ - (0x0B00_0000, "add"), - (0x4B00_0000, "sub"), - (0x0A00_0000, "and"), - (0x6A00_0000, "ands"), // flag-setting and (shift 0) - (0x2A00_0000, "orr"), - (0x4A00_0000, "eor"), - (0x1AC0_0C00, "sdiv"), - (0x1AC0_0800, "udiv"), - (0x1AC0_2000, "lsl"), - (0x1AC0_2400, "lsr"), - (0x1AC0_2800, "asr"), -]; - -/// `ret` (`ret x30`): the exact operandless encoding, matched first. -const RET_WORD: u32 = 0xD65F_0000 | (30 << 5); - -fn decode(out: &mut String, w: u32, ctx: Option>) { - // Exact, operandless / special encodings first. - if w == RET_WORD { - out.push_str("ret"); - return; - } - if w & 0xFFE0_001F == 0xD400_0001 { - let _ = write!(out, "svc #{}", (w >> 5) & 0xFFFF); - return; - } - if w & 0xFFFF_FC1F == 0xD63F_0000 { - out.push_str("blr "); - sp(out, true, rn(w)); - return; - } - - // `mov` (register) is `orr Xd, xzr, Xm` with rn == 31; render it as a move. - if w & 0x7FE0_FFE0 == 0x2A00_03E0 { - out.push_str("mov "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - - // neg Rd, Rm = sub Rd, RZR, Rm (rn == 31), the objdump alias. - if w & 0x7FE0_FC00 == 0x4B00_0000 && rn(w) == 31 { - out.push_str("neg "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - - // The shared three-register form. - for &(m, mnem) in RRR_TABLE { - if w & RRR_MASK == m { - out.push_str(mnem); - out.push(' '); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - } - - // add/sub, shifted-register form with a nonzero LSL shift amount (imm6 at - // bits[15:10]). The shift-0 case is already handled by the table above. - if w & RRR_NO_IMM6_MASK == 0x0B00_0000 || w & RRR_NO_IMM6_MASK == 0x4B00_0000 { - let is_sub = w & RRR_NO_IMM6_MASK == 0x4B00_0000; - out.push_str(if is_sub { "sub " } else { "add " }); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - let _ = write!(out, ", lsl #{}", (w >> 10) & 0x3F); - return; - } - - // madd/msub (mul is madd with ra == 31). - if w & 0x7FE0_8000 == 0x1B00_0000 || w & 0x7FE0_8000 == 0x1B00_8000 { - let is_sub = (w >> 15) & 1 != 0; - if !is_sub && ra(w) == 31 { - out.push_str("mul "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - if is_sub && ra(w) == 31 { - // mneg Rd, Rn, Rm = msub Rd, Rn, Rm, RZR - out.push_str("mneg "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - out.push_str(if is_sub { "msub " } else { "madd " }); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - out.push_str(", "); - gp(out, sf(w), ra(w)); - return; - } - - // add/sub immediate (bit30 = op is sub, bit29 = S sets flags, bit22 = - // shift-by-12). When S is set and rd == 31 the alias is cmp / cmn. Rn/Rd - // 31 otherwise mean sp. - if w & 0x1F80_0000 == 0x1100_0000 { - let is_sub = (w >> 30) & 1 != 0; - let setflags = (w >> 29) & 1 != 0; - let shift12 = (w >> 22) & 1 != 0; - let imm: u32 = (w >> 10) & 0xFFF; - if setflags && rd(w) == 31 { - out.push_str(if is_sub { "cmp " } else { "cmn " }); - sp(out, sf(w), rn(w)); - let _ = write!(out, ", #{imm}"); - if shift12 { - out.push_str(", lsl #12"); - } - return; - } - // mov Rd, Rn (to/from SP) = add Rd, Rn, #0 with no shift. - if !setflags && !is_sub && imm == 0 && !shift12 && (rd(w) == 31 || rn(w) == 31) { - out.push_str("mov "); - sp(out, sf(w), rd(w)); - out.push_str(", "); - sp(out, sf(w), rn(w)); - return; - } - let mnem = if setflags { - if is_sub { "subs" } else { "adds" } - } else if is_sub { - "sub" - } else { - "add" - }; - out.push_str(mnem); - out.push(' '); - sp(out, sf(w), rd(w)); - out.push_str(", "); - sp(out, sf(w), rn(w)); - let _ = write!(out, ", #{imm}"); - if shift12 { - out.push_str(", lsl #12"); - } - return; - } - - // movn (move wide negated): rendered as `mov Rd, #<~imm>`, matching - // objdump's alias. - if w & 0x7F80_0000 == 0x1280_0000 { - let imm: u64 = ((w >> 5) & 0xFFFF) as u64; - let shift: u32 = ((w >> 21) & 3) * 16; - let val = !(imm << shift); - out.push_str("mov "); - gp(out, sf(w), rd(w)); - if sf(w) { - let _ = write!(out, ", #0x{val:x}"); - } else { - let _ = write!(out, ", #0x{:x}", val & 0xFFFF_FFFF); - } - return; - } - - // movz / movk (32/64-bit, with an optional lsl of the 16-bit immediate). - // objdump aliases movz to `mov Rd, #>`, so render the - // assembled value for movz. - if w & 0x7F80_0000 == 0x5280_0000 || w & 0x7F80_0000 == 0x7280_0000 { - let keep = (w >> 29) & 1 != 0; // opc 11 -> movk, 10 -> movz - let imm: u64 = ((w >> 5) & 0xFFFF) as u64; - let shift: u32 = ((w >> 21) & 3) * 16; - if !keep { - // movz -> mov Rd, #value - let val = imm << shift; - out.push_str("mov "); - gp(out, sf(w), rd(w)); - if sf(w) { - let _ = write!(out, ", #0x{val:x}"); - } else { - let _ = write!(out, ", #0x{:x}", val & 0xFFFF_FFFF); - } - return; - } - out.push_str("movk "); - gp(out, sf(w), rd(w)); - let _ = write!(out, ", #{imm}"); - if shift != 0 { - let _ = write!(out, ", lsl #{shift}"); - } - return; - } - - // adds/subs (shifted register, shift 0), with cmn/cmp aliases when - // rd == 31. - if w & 0x7FE0_FC00 == 0x2B00_0000 || w & 0x7FE0_FC00 == 0x6B00_0000 { - let is_sub = (w >> 30) & 1 != 0; - if rd(w) == 31 { - out.push_str(if is_sub { "cmp " } else { "cmn " }); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - if is_sub && rn(w) == 31 { - // negs Rd, Rm = subs Rd, RZR, Rm - out.push_str("negs "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - out.push_str(if is_sub { "subs " } else { "adds " }); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - return; - } - - // cset (csinc Rd, zr, zr, invert(cond)). The mask excludes the cond field - // [12:15]. - if w & 0x7FFF_0FE0 == 0x1A9F_07E0 { - out.push_str("cset "); - gp(out, sf(w), rd(w)); - let _ = write!(out, ", {}", cond_name(((w >> 12) & 0xF) ^ 1)); - return; - } - - // csel Rd, Rn, Rm, cond. - if w & 0x7FE0_0C00 == 0x1A80_0000 { - out.push_str("csel "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - let _ = write!(out, ", {}", cond_name((w >> 12) & 0xF)); - return; - } - - // cbz / cbnz Rt, label (bit24 selects nz). - if w & 0x7E00_0000 == 0x3400_0000 { - let nz = (w >> 24) & 1 != 0; - let imm19: i32 = ((w << 8) as i32) >> 13; // sign-extend bits[5:23] - out.push_str(if nz { "cbnz " } else { "cbz " }); - gp(out, sf(w), rd(w)); - out.push_str(", "); - render_target(out, ctx, imm19 as i64 * 4); - return; - } - - // tbz / tbnz Rt, #bit, label (bit24 selects nz). Test-bit index is b5:b40. - if w & 0x7E00_0000 == 0x3600_0000 { - let nz = (w >> 24) & 1 != 0; - let bit: u32 = (((w >> 31) & 1) << 5) | ((w >> 19) & 0x1F); - let imm14: i32 = (((w << 13) as i32) >> 18) * 4; // bits[18:5], scaled - out.push_str(if nz { "tbnz " } else { "tbz " }); - gp(out, (w >> 31) & 1 != 0, rd(w)); - let _ = write!(out, ", #{bit}, "); - render_target(out, ctx, imm14 as i64); - return; - } - - // b.cond label (bit4 == 0 distinguishes it from the - // consistent-conditional-branch forms). - if w & 0xFF00_0010 == 0x5400_0000 { - let imm19: i32 = (((w << 8) as i32) >> 13) * 4; - let _ = write!(out, "b.{} ", cond_name(w & 0xF)); - render_target(out, ctx, imm19 as i64); - return; - } - - // b / bl label (imm26 << 2). - if w & 0xFC00_0000 == 0x1400_0000 || w & 0xFC00_0000 == 0x9400_0000 { - let link = (w >> 31) & 1 != 0; - let imm26: i32 = (((w << 6) as i32) >> 6) << 2; - out.push_str(if link { "bl " } else { "b " }); - render_target(out, ctx, imm26 as i64); - return; - } - - // stp/ldp pair, pre/post-index (64-bit). - if w & 0xFFC0_0000 == 0xA980_0000 || w & 0xFFC0_0000 == 0xA8C0_0000 { - let load = (w >> 22) & 1 != 0; - let imm7: i32 = (((w << 10) as i32) >> 25) * 8; // bits[15:21], scaled by 8 - out.push_str(if load { "ldp " } else { "stp " }); - gp(out, true, rd(w)); - out.push_str(", "); - gp(out, true, ra(w)); // rt2 is bits[10:15] - out.push_str(", ["); - sp(out, true, rn(w)); - if load { - // post-index: `[base], #imm` - let _ = write!(out, "], #{imm7}"); - } else { - // pre-index: `[base, #imm]!` - let _ = write!(out, ", #{imm7}]!"); - } - return; - } - - // Logical immediate: and/orr/eor/ands, with mov (orr from zr) and tst - // (ands to zr) aliases. - if w & 0x1F80_0000 == 0x1200_0000 { - let opc = (w >> 29) & 3; - let nn: u32 = (w >> 22) & 1; - let immr: u32 = (w >> 16) & 0x3F; - let imms: u32 = (w >> 10) & 0x3F; - if let Some(val) = decode_bitmask(sf(w), nn, imms, immr) { - if opc == 1 && rn(w) == 31 { - // mov Rd, #imm - out.push_str("mov "); - gp(out, sf(w), rd(w)); - let _ = write!(out, ", #0x{val:x}"); - return; - } - if opc == 3 && rd(w) == 31 { - // tst Rn, #imm - out.push_str("tst "); - gp(out, sf(w), rn(w)); - let _ = write!(out, ", #0x{val:x}"); - return; - } - let mnem = match opc { - 0 => "and", - 1 => "orr", - 2 => "eor", - _ => "ands", - }; - out.push_str(mnem); - out.push(' '); - sp(out, sf(w), rd(w)); // and/orr/eor immediate can target sp - out.push_str(", "); - gp(out, sf(w), rn(w)); - let _ = write!(out, ", #0x{val:x}"); - return; - } - } - - // extr Rd, Rn, Rm, #lsb (ror when rn == rm). bit23 == 1 separates it from - // the bitfield ops. - if w & 0x7F80_0000 == 0x1380_0000 { - let lsb: u32 = (w >> 10) & 0x3F; - if rn(w) == rm(w) { - out.push_str("ror "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - let _ = write!(out, ", #{lsb}"); - } else { - out.push_str("extr "); - gp(out, sf(w), rd(w)); - out.push_str(", "); - gp(out, sf(w), rn(w)); - out.push_str(", "); - gp(out, sf(w), rm(w)); - let _ = write!(out, ", #{lsb}"); - } - return; - } - - // STUR/LDUR: unscaled signed 9-bit offset (integer). size 00=b 01=h 10=w - // 11=x; opc picks store / load / sign-extending load. - if w & 0x3F20_0C00 == 0x3800_0000 { - let size = w >> 30; - let opc = (w >> 22) & 3; - let imm9: i32 = ((w << 11) as i32) >> 23; // bits[20:12], sign-extended - let load = opc != 0; - let signed = opc >= 2; - let to_x = size == 3 || opc == 2; // x-sized access, or sign-extend to 64-bit - let suffix: &str = match size { - 0 => "b", - 1 => "h", - 2 if signed => "w", - _ => "", - }; - out.push_str(if load { "ldur" } else { "stur" }); - if signed { - out.push('s'); - } - out.push_str(suffix); - out.push(' '); - gp(out, to_x, rd(w)); - out.push_str(", ["); - sp(out, true, rn(w)); - if imm9 != 0 { - let _ = write!(out, ", #{imm9}]"); - } else { - out.push(']'); - } - return; - } - - // STUR/LDUR (FP/SIMD), unscaled signed 9-bit offset. Width is b/h/s/d by - // size, or q (128-bit) when opc bit1 is set. - if w & 0x3F20_0C00 == 0x3C00_0000 { - let size = w >> 30; - let opc = (w >> 22) & 3; - let imm9: i32 = ((w << 11) as i32) >> 23; - let load = opc & 1 != 0; - let c: char = if opc & 2 != 0 { - 'q' - } else { - match size { - 0 => 'b', - 1 => 'h', - 2 => 's', - _ => 'd', - } - }; - out.push_str(if load { "ldur " } else { "stur " }); - let _ = write!(out, "{c}{}, [", rd(w)); - sp(out, true, rn(w)); - if imm9 != 0 { - let _ = write!(out, ", #{imm9}]"); - } else { - out.push(']'); - } - return; - } - - // umulh / smulh (unsigned/signed 64x64 -> high 64). Ra field is 31. - if w & 0xFFE0_FC00 == 0x9BC0_7C00 || w & 0xFFE0_FC00 == 0x9B40_7C00 { - let uns = (w >> 23) & 1 != 0; - let _ = write!( - out, - "{} x{}, x{}, x{}", - if uns { "umulh" } else { "smulh" }, - rd(w), - rn(w), - rm(w) - ); - return; - } - - // Bitfield move (SBFM/UBFM/BFM) with the usual aliases. - if w & 0x1F80_0000 == 0x1300_0000 && (w >> 29) & 3 != 3 { - bitfield(out, w); - return; - } - - // stp/ldp signed offset (bit31 selects 32/64-bit, bit22 selects load). - if w & 0x7FC0_0000 == 0x2900_0000 || w & 0x7FC0_0000 == 0x2940_0000 { - let x64 = (w >> 31) & 1 != 0; - let load = (w >> 22) & 1 != 0; - let scale: u32 = if x64 { 3 } else { 2 }; - let imm7: i32 = (((w << 10) as i32) >> 25) << scale; // bits[21:15], scaled - out.push_str(if load { "ldp " } else { "stp " }); - gp(out, x64, rd(w)); - out.push_str(", "); - gp(out, x64, ra(w)); // rt2 is bits[14:10] - out.push_str(", ["); - sp(out, true, rn(w)); - if imm7 != 0 { - let _ = write!(out, ", #{imm7}]"); - } else { - out.push(']'); - } - return; - } - - // adr / adrp Rd, imm (PC-relative address; adrp is page-scaled). - if w & 0x1F00_0000 == 0x1000_0000 { - let page = (w >> 31) & 1 != 0; - let immlo: i64 = ((w >> 29) & 3) as i64; - let immhi: i64 = ((((w & 0x00FF_FFE0) << 8) as i32) >> 13) as i64; // bits[23:5], sign-extended - let mut imm: i64 = (immhi << 2) | immlo; - if page { - imm <<= 12; - } - out.push_str(if page { "adrp " } else { "adr " }); - gp(out, true, rd(w)); - if let Some(c) = ctx { - // absolute target: adrp is page-based, adr is byte-based - let pc_base: u64 = if page { c.pc & !0xFFF } else { c.pc }; - let abs: u64 = pc_base.wrapping_add(imm as u64); - let _ = write!(out, ", 0x{abs:x}"); - annotate_sym(out, c.syms, abs); - } else if imm < 0 { - let _ = write!(out, ", .-0x{:x}", (-imm) as u64); - } else { - let _ = write!(out, ", .+0x{imm:x}"); - } - return; - } - - if mem_imm(out, w) { - return; - } - if fp_decode(out, w) { - return; - } - if vec_decode(out, w) { - return; - } - - // Unknown: emit the raw word so nothing is silently lost. - let _ = write!(out, ".word 0x{w:08x}"); -} - -/// Decode a logical-immediate `(N:immr:imms)` to its 32/64-bit value, or -/// `None` if the encoding is reserved. This is ARM's `DecodeBitMasks` -/// (immediate result only): a run of `s+1` ones, rotated right by `r` within -/// an `esize`-bit element, replicated across the register. -fn decode_bitmask(x64: bool, n: u32, imms: u32, immr: u32) -> Option { - let width: u32 = if x64 { 64 } else { 32 }; - // len = position of the highest set bit of (N : NOT(imms)), a 7-bit quantity. - let combined: u32 = ((n & 1) << 6) | ((!imms) & 0x3F); - if combined == 0 { - return None; - } - let mut len: u32 = 6; - while len > 0 && (combined & (1 << len)) == 0 { - len -= 1; - } - if len == 0 { - return None; // element size 1 is not a valid logical immediate - } - let esize: u32 = 1 << len; - if esize > width { - return None; // N == 1 is only valid for the 64-bit form - } - let levels: u32 = esize - 1; - let s: u32 = imms & levels; - let r: u32 = immr & levels; - if s == levels { - return None; // an all-ones element is reserved - } - let elem: u64 = (1u64 << (s + 1)) - 1; - let rotated = ror_in_esize(elem, r, esize); - let mut result: u64 = 0; - let mut pos: u32 = 0; - while pos < width { - result |= rotated << pos; - pos += esize; - } - Some(if x64 { result } else { result & 0xFFFF_FFFF }) -} - -/// Rotate the low `esize` bits of `v` right by `r`. -fn ror_in_esize(v: u64, r: u32, esize: u32) -> u64 { - let mask: u64 = if esize == 64 { - u64::MAX - } else { - (1u64 << esize) - 1 - }; - let rr = r % esize; - if rr == 0 { - return v & mask; - } - ((v >> rr) | (v << (esize - rr))) & mask -} - -/// Scaled-unsigned-offset loads/stores (integer and FP). Returns whether it -/// matched. -fn mem_imm(out: &mut String, w: u32) -> bool { - struct Entry { - match_word: u32, - mnem: &'static str, - x64: bool, - scale: u32, - fp: bool, - dbl: bool, - } - const TABLE: &[Entry] = &[ - Entry { - match_word: 0xF900_0000, - mnem: "str", - x64: true, - scale: 3, - fp: false, - dbl: false, - }, - Entry { - match_word: 0xF940_0000, - mnem: "ldr", - x64: true, - scale: 3, - fp: false, - dbl: false, - }, - Entry { - match_word: 0xB900_0000, - mnem: "str", - x64: false, - scale: 2, - fp: false, - dbl: false, - }, - Entry { - match_word: 0xB940_0000, - mnem: "ldr", - x64: false, - scale: 2, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x3900_0000, - mnem: "strb", - x64: false, - scale: 0, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x3940_0000, - mnem: "ldrb", - x64: false, - scale: 0, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x39C0_0000, - mnem: "ldrsb", - x64: false, - scale: 0, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x7900_0000, - mnem: "strh", - x64: false, - scale: 1, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x7940_0000, - mnem: "ldrh", - x64: false, - scale: 1, - fp: false, - dbl: false, - }, - Entry { - match_word: 0x79C0_0000, - mnem: "ldrsh", - x64: false, - scale: 1, - fp: false, - dbl: false, - }, - Entry { - match_word: 0xBD00_0000, - mnem: "str", - x64: false, - scale: 2, - fp: true, - dbl: false, - }, - Entry { - match_word: 0xBD40_0000, - mnem: "ldr", - x64: false, - scale: 2, - fp: true, - dbl: false, - }, - Entry { - match_word: 0xFD00_0000, - mnem: "str", - x64: false, - scale: 3, - fp: true, - dbl: true, - }, - Entry { - match_word: 0xFD40_0000, - mnem: "ldr", - x64: false, - scale: 3, - fp: true, - dbl: true, - }, - Entry { - match_word: 0x3D80_0000, - mnem: "str", - x64: false, - scale: 4, - fp: true, - dbl: false, - }, // Q - Entry { - match_word: 0x3DC0_0000, - mnem: "ldr", - x64: false, - scale: 4, - fp: true, - dbl: false, - }, // Q - Entry { - match_word: 0x7D00_0000, - mnem: "str", - x64: false, - scale: 1, - fp: true, - dbl: false, - }, // H (f16) - Entry { - match_word: 0x7D40_0000, - mnem: "ldr", - x64: false, - scale: 1, - fp: true, - dbl: false, - }, // H (f16) - ]; - for m in TABLE { - if w & 0xFFC0_0000 != m.match_word { - continue; - } - let off: u32 = ((w >> 10) & 0xFFF) << m.scale; - out.push_str(m.mnem); - out.push(' '); - if m.fp { - // The FP view: Q (128-bit), D (64-bit), H (16-bit, the f16 - // boundary form), else S. - let c: char = if m.scale == 4 { - 'q' - } else if m.dbl { - 'd' - } else if m.scale == 1 { - 'h' - } else { - 's' - }; - let _ = write!(out, "{c}{}", rd(w)); - } else { - gp(out, m.x64, rd(w)); - } - out.push_str(", ["); - sp(out, true, rn(w)); - if off != 0 { - let _ = write!(out, ", #{off}]"); - } else { - out.push(']'); - } - return true; - } - // strb/ldrb/etc with a zero-offset `[xn]` form already covered (off field 0). - false -} - -/// Scalar floating-point instructions. Returns whether it matched. -fn fp_decode(out: &mut String, w: u32) -> bool { - let dbl = (w >> 22) & 1 != 0; - let fc: char = if dbl { 'd' } else { 's' }; - // Three-register scalar FP arithmetic. - const FP3: &[(u32, &str)] = &[ - (0x1E20_2800, "fadd"), - (0x1E20_3800, "fsub"), - (0x1E20_0800, "fmul"), - (0x1E20_1800, "fdiv"), - ]; - for &(m, mnem) in FP3 { - if w & 0xFFA0_FC00 == m { - let _ = write!(out, "{mnem} {fc}{}, {fc}{}, {fc}{}", rd(w), rn(w), rm(w)); - return true; - } - } - // Floating-point data-processing (3-source): Rd = op(Rn*Rm, Ra). Mask - // excludes ftype (bit 22, read via `dbl` above) and the four 5-bit - // register fields. - const FP3A: &[(u32, &str)] = &[ - (0x1F00_0000, "fmadd"), - (0x1F00_8000, "fmsub"), - (0x1F20_0000, "fnmadd"), - (0x1F20_8000, "fnmsub"), - ]; - for &(m, mnem) in FP3A { - if w & 0xFF20_8000 == m { - let _ = write!( - out, - "{mnem} {fc}{}, {fc}{}, {fc}{}, {fc}{}", - rd(w), - rn(w), - rm(w), - ra(w) - ); - return true; - } - } - if w & 0xFFA0_0C00 == 0x1E20_0C00 { - // fcsel (mask excludes the cond field [12:15]) - let _ = write!( - out, - "fcsel {fc}{}, {fc}{}, {fc}{}, {}", - rd(w), - rn(w), - rm(w), - cond_name((w >> 12) & 0xF) - ); - return true; - } - if w & 0xFFA0_FC1F == 0x1E20_2000 { - // fcmp - let _ = write!(out, "fcmp {fc}{}, {fc}{}", rn(w), rm(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E60_4000 { - // fmov (reg, 64-bit view) - let _ = write!(out, "fmov d{}, d{}", rd(w), rn(w)); - return true; - } - // fmov between a general register and an FP register (both directions, - // s/d). - struct Fmov { - match_word: u32, - to_gpr: bool, - x64: bool, - } - const FMOV: &[Fmov] = &[ - Fmov { - match_word: 0x1E27_0000, - to_gpr: false, - x64: false, - }, - Fmov { - match_word: 0x9E67_0000, - to_gpr: false, - x64: true, - }, - Fmov { - match_word: 0x1E26_0000, - to_gpr: true, - x64: false, - }, - Fmov { - match_word: 0x9E66_0000, - to_gpr: true, - x64: true, - }, - ]; - for e in FMOV { - if w & 0xFFFF_FC00 == e.match_word { - let f: char = if e.x64 { 'd' } else { 's' }; - out.push_str("fmov "); - if e.to_gpr { - gp(out, e.x64, rd(w)); - let _ = write!(out, ", {f}{}", rn(w)); - } else { - let _ = write!(out, "{f}{}, ", rd(w)); - gp(out, e.x64, rn(w)); - } - return true; - } - } - // Two-register scalar FP ops (sqrt, rounding, conversions). - const FP2: &[(u32, &str)] = &[ - (0x1E21_C000, "fsqrt"), - (0x1E24_4000, "frintn"), - (0x1E24_C000, "frintp"), - (0x1E25_4000, "frintm"), - (0x1E25_C000, "frintz"), - ]; - for &(m, mnem) in FP2 { - if w & 0xFFBF_FC00 == m { - let _ = write!(out, "{mnem} {fc}{}, {fc}{}", rd(w), rn(w)); - return true; - } - } - if w & 0xFFBF_FC00 == 0x1E22_0000 { - // scvtf sd, wn - let _ = write!(out, "scvtf {fc}{}, ", rd(w)); - gp(out, false, rn(w)); - return true; - } - if w & 0xFFBF_FC00 == 0x1E23_0000 { - // ucvtf - let _ = write!(out, "ucvtf {fc}{}, ", rd(w)); - gp(out, false, rn(w)); - return true; - } - if w & 0xFFBF_FC00 == 0x1E38_0000 { - // fcvtzs wd, sn - out.push_str("fcvtzs "); - gp(out, false, rd(w)); - let _ = write!(out, ", {fc}{}", rn(w)); - return true; - } - if w & 0xFFBF_FC00 == 0x1E39_0000 { - // fcvtzu - out.push_str("fcvtzu "); - gp(out, false, rd(w)); - let _ = write!(out, ", {fc}{}", rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E22_C000 { - // fcvt d, s (s->d) - let _ = write!(out, "fcvt d{}, s{}", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E62_4000 { - // fcvt s, d (d->s) - let _ = write!(out, "fcvt s{}, d{}", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1EE2_4000 { - // fcvt s, h (h->s, f16 widen) - let _ = write!(out, "fcvt s{}, h{}", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E23_C000 { - // fcvt h, s (s->h, f16 narrow) - let _ = write!(out, "fcvt h{}, s{}", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x1E63_C000 { - // fcvt h, d (d->h, f16 narrow, single round) - let _ = write!(out, "fcvt h{}, d{}", rd(w), rn(w)); - return true; - } - false -} - -/// NEON (128-bit vector) instructions. Returns whether it matched. -fn vec_decode(out: &mut String, w: u32) -> bool { - // Three-register 4S / 16B forms. - struct V3 { - match_word: u32, - mnem: &'static str, - b16: bool, - } - const V3_TABLE: &[V3] = &[ - V3 { - match_word: 0x4E20_D400, - mnem: "fadd", - b16: false, - }, - V3 { - match_word: 0x4EA0_D400, - mnem: "fsub", - b16: false, - }, - V3 { - match_word: 0x6E20_DC00, - mnem: "fmul", - b16: false, - }, - V3 { - match_word: 0x6E20_FC00, - mnem: "fdiv", - b16: false, - }, - V3 { - match_word: 0x4EA0_F400, - mnem: "fmin", - b16: false, - }, - V3 { - match_word: 0x4E20_F400, - mnem: "fmax", - b16: false, - }, - V3 { - match_word: 0x4E20_E400, - mnem: "fcmeq", - b16: false, - }, - V3 { - match_word: 0x6EA0_E400, - mnem: "fcmgt", - b16: false, - }, - V3 { - match_word: 0x6E20_E400, - mnem: "fcmge", - b16: false, - }, - V3 { - match_word: 0x6E60_1C00, - mnem: "bsl", - b16: true, - }, - V3 { - match_word: 0x4E20_CC00, - mnem: "fmla", - b16: false, - }, - V3 { - match_word: 0x4EA0_CC00, - mnem: "fmls", - b16: false, - }, - V3 { - match_word: 0x6E20_D400, - mnem: "faddp", - b16: false, - }, - ]; - for e in V3_TABLE { - if w == (e.match_word | (rm(w) << 16) | (rn(w) << 5) | rd(w)) { - let s = if e.b16 { "16b" } else { "4s" }; - let _ = write!( - out, - "{} v{}.{s}, v{}.{s}, v{}.{s}", - e.mnem, - rd(w), - rn(w), - rm(w) - ); - return true; - } - } - // Two-register vector forms. - if w & 0xFFFF_FC00 == 0x6EA0_F800 { - return vec2(out, w, "fneg", "4s"); - } - if w & 0xFFFF_FC00 == 0x6EA1_F800 { - return vec2(out, w, "fsqrt", "4s"); - } - if w & 0xFFFF_FC00 == 0x6E20_5800 { - return vec2(out, w, "mvn", "16b"); - } - // Horizontal reduce: addv sums the 4S lanes to a scalar S register; - // faddpScalar finishes a pairwise float reduction from a 2S pair. Both - // cross from the Vn.4S/2S view into Sd. - if w & 0xFFFF_FC00 == 0x4EB1_B800 { - let _ = write!(out, "addv s{}, v{}.4s", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x7E30_D800 { - let _ = write!(out, "faddp s{}, v{}.2s", rd(w), rn(w)); - return true; - } - // mov vd.16b, vn.16b is orr with rm == rn. - if w & 0xFFE0_FC00 == 0x4EA0_1C00 && rm(w) == rn(w) { - let _ = write!(out, "mov v{}.16b, v{}.16b", rd(w), rn(w)); - return true; - } - if w & 0xFFFF_FC00 == 0x4E04_0C00 { - // dup Vd.4S, Wn - let _ = write!(out, "dup v{}.4s, ", rd(w)); - gp(out, false, rn(w)); - return true; - } - // dup Vd.4S, Vn.s[i] / dup Sd, Vn.s[i] / ins Vd.s[i], Vn.s[0]: imm5 in - // bits[16:20]. - let imm5 = (w >> 16) & 0x1F; - let lane = (imm5 >> 3) & 3; - if w & 0xFFE0_FC00 == 0x4E00_0400 { - let _ = write!(out, "dup v{}.4s, v{}.s[{lane}]", rd(w), rn(w)); - return true; - } - if w & 0xFFE0_FC00 == 0x5E00_0400 { - let _ = write!(out, "dup s{}, v{}.s[{lane}]", rd(w), rn(w)); - return true; - } - if w & 0xFFE0_8C00 == 0x6E00_0400 { - let _ = write!(out, "ins v{}.s[{lane}], v{}.s[0]", rd(w), rn(w)); - return true; - } - if w & 0xFFE0_FC00 == 0x0E00_3C00 { - // umov Wd, Vn.s[i] - out.push_str("umov "); - gp(out, false, rd(w)); - let _ = write!(out, ", v{}.s[{lane}]", rn(w)); - return true; - } - if w & 0xFFE0_FC00 == 0x4E00_1C00 { - // ins Vd.s[i], Wn - let _ = write!(out, "ins v{}.s[{lane}], ", rd(w)); - gp(out, false, rn(w)); - return true; - } - false -} - -fn vec2(out: &mut String, w: u32, mnem: &str, suffix: &str) -> bool { - let _ = write!(out, "{mnem} v{}.{suffix}, v{}.{suffix}", rd(w), rn(w)); - true -} - -/// Render a bitfield-move (SBFM/UBFM/BFM) using the alias a debugger expects -/// (uxt*/sxt*/lsl/lsr/asr/ubfx/sbfx/bfi/bfxil). `opc` selects the family; -/// `immr`/`imms` drive the alias choice. -fn bitfield(out: &mut String, w: u32) { - let opc = (w >> 29) & 3; - let x64 = sf(w); - let width: u32 = if x64 { 64 } else { 32 }; - let immr: u32 = (w >> 16) & 0x3F; - let imms: u32 = (w >> 10) & 0x3F; - - match opc { - 2 => { - // UBFM - if imms + 1 == immr && imms != width - 1 { - // lsl #(width-1-imms) - bf_two(out, "lsl", x64, rd(w), rn(w)); - let _ = write!(out, ", #{}", width - 1 - imms); - } else if imms == width - 1 { - // lsr #immr - bf_two(out, "lsr", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}"); - } else if immr == 0 && imms == 7 && !x64 { - bf_two(out, "uxtb", false, rd(w), rn(w)); - } else if immr == 0 && imms == 15 && !x64 { - bf_two(out, "uxth", false, rd(w), rn(w)); - } else if imms < immr { - // ubfiz Rd, Rn, #(width-immr), #(imms+1) - bf_two(out, "ubfiz", x64, rd(w), rn(w)); - let _ = write!(out, ", #{}, #{}", width - immr, imms + 1); - } else { - // ubfx Rd, Rn, #immr, #(imms-immr+1) - bf_two(out, "ubfx", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}, #{}", imms - immr + 1); - } - } - 0 => { - // SBFM - if imms == width - 1 { - // asr #immr - bf_two(out, "asr", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}"); - } else if immr == 0 && imms == 7 { - bf_two(out, "sxtb", x64, rd(w), rn(w)); - } else if immr == 0 && imms == 15 { - bf_two(out, "sxth", x64, rd(w), rn(w)); - } else if immr == 0 && imms == 31 && x64 { - // sxtw Xd, Wn - out.push_str("sxtw "); - gp(out, true, rd(w)); - out.push_str(", "); - gp(out, false, rn(w)); - } else if imms < immr { - // sbfiz - bf_two(out, "sbfiz", x64, rd(w), rn(w)); - let _ = write!(out, ", #{}, #{}", width - immr, imms + 1); - } else { - bf_two(out, "sbfx", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}, #{}", imms - immr + 1); - } - } - _ => { - // BFM - if imms < immr { - // bfi Rd, Rn, #(width-immr), #(imms+1) - bf_two(out, "bfi", x64, rd(w), rn(w)); - let _ = write!(out, ", #{}, #{}", width - immr, imms + 1); - } else { - // bfxil Rd, Rn, #immr, #(imms-immr+1) - bf_two(out, "bfxil", x64, rd(w), rn(w)); - let _ = write!(out, ", #{immr}, #{}", imms - immr + 1); - } - } - } -} - -/// The common `mnem Rd, Rn` shape for bitfield aliases, at the register width. -fn bf_two(out: &mut String, mnem: &str, xw: bool, d: u32, n: u32) { - out.push_str(mnem); - out.push(' '); - gp(out, xw, d); - out.push_str(", "); - gp(out, xw, n); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn round_trips_basic_words() { - // Hand-computed words (no dependency on the encoder module): - // ret = 0xD65F0000 | (30 << 5); add w0,w1,w2; bl .+8; nop. - assert_eq!(one(0xD65F_03C0), "ret"); - assert_eq!(one(0x0B02_0020), "add w0, w1, w2"); - assert_eq!(one(0x8B02_0020), "add x0, x1, x2"); - assert_eq!(one(0x9400_0002), "bl .+8"); - assert_eq!(one(0x1400_0002), "b .+8"); - // nop is outside the emitted-subset decode coverage: it must surface - // as a raw word, never a mis-decode. - assert_eq!(one(0xD503_201F), ".word 0xd503201f"); - } - - #[test] - fn decodes_objdump_vectors() { - // Spot checks from the Zig suite (verified against objdump there). - assert_eq!(one(0xEB09_0108), "subs x8, x8, x9"); - assert_eq!(one(0x3400_00E8), "cbz w8, .+28"); - assert_eq!(one(0x5400_00C3), "b.cc .+24"); - assert_eq!(one(0xF85F_83A8), "ldur x8, [x29, #-8]"); - assert_eq!(one(0xA941_7BFD), "ldp x29, x30, [sp, #16]"); - assert_eq!(one(0x5300_3D08), "uxth w8, w8"); - assert_eq!(one(0x531F_7908), "lsl w8, w8, #1"); - assert_eq!(one(0x9340_7C00), "sxtw x0, w0"); - assert_eq!(one(0x9280_0AA1), "mov x1, #0xffffffffffffffaa"); - } - - #[test] - fn format_with_lines_interleaves_markers() { - let code = [0x0B02_0020, 0x1B01_7C00, 0xD65F_03C0]; - let lines = [ - SourceLine { offset: 0, line: 2 }, - SourceLine { offset: 4, line: 3 }, - ]; - assert_eq!( - format_with_lines(&code, &lines), - "; line 2\n\ - 0000: 0b020020 add w0, w1, w2\n\ - ; line 3\n\ - 0004: 1b017c00 mul w0, w0, w1\n\ - 0008: d65f03c0 ret\n" - ); - } - - #[test] - fn format_module_annotates_branch_targets() { - // fn f at word 0, fn g at word 3; b .+8 lands at words 2 and 4. - let code = [ - 0x1400_0002, - 0xD65F_03C0, - 0x1400_0002, - 0xD65F_03C0, - 0xD65F_03C0, - ]; - let syms = [Sym { name: "f", word: 0 }, Sym { name: "g", word: 3 }]; - let text = format_module(&code, &syms); - assert!(text.contains("b .+8 "), "{text}"); - assert!(text.contains("b .+8 "), "{text}"); - } - - #[test] - fn format_elf_resolves_absolute_targets() { - let base: u64 = 0x1000; - let code = [0x9400_0002, 0xB000_0000, 0xD65F_03C0]; - let syms = [ - AddrSym { - name: "start", - addr: 0x1000, - size: 8, - is_func: true, - }, - AddrSym { - name: "callee", - addr: 0x1008, - size: 4, - is_func: true, - }, - AddrSym { - name: "gv", - addr: 0x2000, - size: 16, - is_func: false, - }, - ]; - let text = format_elf(&code, base, &syms); - assert!(text.contains("0000000000001000 :"), "{text}"); - assert!(text.contains("0000000000001008 :"), "{text}"); - assert!(text.contains("bl 0x1008 "), "{text}"); - assert!(text.contains("adrp x0, 0x2000 "), "{text}"); - } -} diff --git a/src/target/src/aarch64/encode.rs b/src/target/src/aarch64/encode.rs index 60dc798..ad35daa 100644 --- a/src/target/src/aarch64/encode.rs +++ b/src/target/src/aarch64/encode.rs @@ -451,6 +451,24 @@ pub fn ldrsh_off(rt: Reg, rn: Reg, off: u16) -> u32 { 0x79C00000 | (((off as u32) >> 1) << 10) | (n(rn) << 5) | n(rt) } +/// Load-acquire register: `size` is the access width in bytes (1, 2, 4 or 8); the value is +/// zero-extended into `rt`. Addressing is a bare base register (`[rn]`) only. +pub fn ldar(rt: Reg, rn: Reg, size: u8) -> u32 { + ordered_word(0x08DF_FC00, size) | (n(rn) << 5) | n(rt) +} + +/// Store-release register: `size` is the access width in bytes (1, 2, 4 or 8). +/// Addressing is a bare base register (`[rn]`) only. +pub fn stlr(rt: Reg, rn: Reg, size: u8) -> u32 { + ordered_word(0x089F_FC00, size) | (n(rn) << 5) | n(rt) +} + +/// `base` (the byte-sized form of an `LDAR`/`STLR`) with the size field for `size` bytes. +fn ordered_word(base: u32, size: u8) -> u32 { + debug_assert!(matches!(size, 1 | 2 | 4 | 8), "ldar/stlr size {size}"); + base | (size.trailing_zeros() << 30) +} + pub fn sdiv(rd: Reg, rn: Reg, rm: Reg) -> u32 { 0x1AC00C00 | (n(rm) << 16) | (n(rn) << 5) | n(rd) } @@ -939,7 +957,11 @@ mod tests { } else { (1u64 << size) - 1 }; - let rotated = ((run >> r) | (run << (size - r))) & mask; + let rotated = if r == 0 { + run + } else { + ((run >> r) | (run << (size - r))) & mask + }; let mut value = 0u64; let mut at = 0; while at < if is64 { 64 } else { 32 } { @@ -1234,6 +1256,19 @@ mod tests { assert_eq!(0x297F10A3, ldp_off_w(Reg::X3, Reg::X4, Reg::X5, -8)); } + #[test] + fn load_acquire_store_release_golden() { + // Checked against GNU as. + assert_eq!(0x88dffc20, ldar(Reg::X0, Reg::X1, 4)); + assert_eq!(0xc8dffc62, ldar(Reg::X2, Reg::X3, 8)); + assert_eq!(0x08dffca4, ldar(Reg::X4, Reg::X5, 1)); + assert_eq!(0x48dffce6, ldar(Reg::X6, Reg::X7, 2)); + assert_eq!(0x889ffc20, stlr(Reg::X0, Reg::X1, 4)); + assert_eq!(0xc89ffc62, stlr(Reg::X2, Reg::X3, 8)); + assert_eq!(0x089ffca4, stlr(Reg::X4, Reg::X5, 1)); + assert_eq!(0x489ffce6, stlr(Reg::X6, Reg::X7, 2)); + } + #[test] fn byte_halfword_offset_golden() { assert_eq!(0x39000C20, strb_off(Reg::X0, Reg::X1, 3)); diff --git a/src/target/src/aarch64/isel_tests.rs b/src/target/src/aarch64/isel_tests.rs index 7fce7fc..f832d2a 100644 --- a/src/target/src/aarch64/isel_tests.rs +++ b/src/target/src/aarch64/isel_tests.rs @@ -1,36 +1,65 @@ // Rust selector boundary tests for the Vulcan-derived AArch64 port (Apache-2.0). use super::*; +use volt_ir::attribute::Endianness; use volt_ir::function::{ - Arith, BinOp, CmpOp, Compare, EdgeDesc, Function, Load, Opcode, Ret, Terminator, + Arith, AtomicOrdering, BinOp, Block, CmpOp, Compare, Convert, ConvertKind, Dot, EdgeDesc, + Extract, Function, Load, MemFlags, Opcode, Ret, Terminator, Value, }; -use volt_ir::types::{FloatKind, IntDesc, Type, TypeKind}; +use volt_ir::types::{FloatKind, IntDesc, Type, TypeKind, VectorDesc}; -fn integer(f: &mut Function, signed: bool, bits: u16) -> Type { - f.types.intern(TypeKind::Int(IntDesc { signed, bits })) +fn integer(f: &mut Function, bits: u16) -> Type { + f.types.intern(TypeKind::Int(IntDesc { bits })) +} + +fn convert(f: &mut Function, b: Block, ty: Type, value: Value, kind: ConvertKind) -> Value { + f.append_inst(b, ty, Opcode::Convert(Convert { value, kind })) +} + +/// `x` (an integer of `from` bits) widened to the 64-bit type `wide`, by sign when `signed`; +/// a 64-bit value is already that wide. +fn widen(f: &mut Function, b: Block, wide: Type, x: Value, from: u16, signed: bool) -> Value { + if from == 64 { + return x; + } + let kind = if signed { + ConvertKind::Sext + } else { + ConvertKind::Zext + }; + convert(f, b, wide, x, kind) +} + +/// The low `bits` of the 64-bit `x`. +fn narrow(f: &mut Function, b: Block, ty: Type, x: Value, bits: u16) -> Value { + if bits == 64 { + return x; + } + convert(f, b, ty, x, ConvertKind::Trunc) } #[test] fn emits_native_divide_remainder_and_high_product() { - for signed in [false, true] { - for op in [BinOp::Div, BinOp::Rem, BinOp::Mulh] { - let mut f = Function::new(); - let ty = integer(&mut f, signed, 64); - let b = f.append_block(); - let a = f.append_block_param(b, ty); - let c = f.append_block_param(b, ty); - let result = f.append_inst(b, ty, Opcode::Arith(Arith { op, lhs: a, rhs: c })); - f.set_terminator(b, Terminator::Ret(Ret::one(result))); - let code = compile(&f).unwrap(); - assert_eq!(code.last(), Some(&crate::aarch64::encode::ret())); - let opcode = if op == BinOp::Mulh { - if signed { 0x9b407c00 } else { 0x9bc07c00 } - } else if signed { - 0x9ac00c00 - } else { - 0x9ac00800 - }; - assert!(code.iter().any(|w| w & 0xffe0fc00 == opcode)); - } + for (op, opcode) in [ + (BinOp::UDiv, 0x9ac00800), + (BinOp::SDiv, 0x9ac00c00), + (BinOp::URem, 0x9ac00800), + (BinOp::SRem, 0x9ac00c00), + (BinOp::UMulh, 0x9bc07c00), + (BinOp::SMulh, 0x9b407c00), + ] { + let mut f = Function::new(); + let ty = integer(&mut f, 64); + let b = f.append_block(); + let a = f.append_block_param(b, ty); + let c = f.append_block_param(b, ty); + let result = f.append_inst(b, ty, Opcode::Arith(Arith { op, lhs: a, rhs: c })); + f.set_terminator(b, Terminator::Ret(Ret::one(result))); + let code = compile(&f).unwrap(); + assert_eq!(code.last(), Some(&crate::aarch64::encode::ret()), "{op:?}"); + assert!(code.iter().any(|w| w & 0xffe0fc00 == opcode), "{op:?}"); + // A remainder is the divide followed by a multiply-subtract. + let msub = code.iter().any(|w| w & 0xffe08000 == 0x9b008000); + assert_eq!(msub, matches!(op, BinOp::URem | BinOp::SRem), "{op:?}"); } } @@ -55,7 +84,7 @@ fn scalar_half_and_quad_data_survive_selection() { #[test] fn object_calls_keep_owned_relocations() { let mut f = Function::new(); - let ty = integer(&mut f, false, 64); + let ty = integer(&mut f, 64); let b = f.append_block(); let value = f.append_call(b, ty, "external_target", &[]); f.set_terminator(b, Terminator::Ret(Ret::one(value))); @@ -75,24 +104,28 @@ fn mapped(f: &Function) -> crate::native::JittedFunction { #[cfg(target_arch = "aarch64")] #[test] fn division_edges_execute_with_aarch64_semantics() { - for signed in [false, true] { - for op in [BinOp::Div, BinOp::Rem] { - let mut f = Function::new(); - let ty = integer(&mut f, signed, 64); - let b = f.append_block(); - let a = f.append_block_param(b, ty); - let c = f.append_block_param(b, ty); - let result = f.append_inst(b, ty, Opcode::Arith(Arith { op, lhs: a, rhs: c })); - f.set_terminator(b, Terminator::Ret(Ret::one(result))); - let image = mapped(&f); - // SAFETY: function is generated for precisely this AAPCS64 signature. - let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { image.entry() }; - let zero = unsafe { entry(123, 0) }; - assert_eq!(zero, if op == BinOp::Div { 0 } else { 123 }); - if signed { - let overflow = unsafe { entry(i64::MIN as u64, u64::MAX) }; - assert_eq!(overflow, if op == BinOp::Div { i64::MIN as u64 } else { 0 }); - } + for (op, signed) in [ + (BinOp::UDiv, false), + (BinOp::SDiv, true), + (BinOp::URem, false), + (BinOp::SRem, true), + ] { + let mut f = Function::new(); + let ty = integer(&mut f, 64); + let b = f.append_block(); + let a = f.append_block_param(b, ty); + let c = f.append_block_param(b, ty); + let result = f.append_inst(b, ty, Opcode::Arith(Arith { op, lhs: a, rhs: c })); + f.set_terminator(b, Terminator::Ret(Ret::one(result))); + let image = mapped(&f); + // SAFETY: function is generated for precisely this AAPCS64 signature. + let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { image.entry() }; + let is_div = matches!(op, BinOp::UDiv | BinOp::SDiv); + let zero = unsafe { entry(123, 0) }; + assert_eq!(zero, if is_div { 0 } else { 123 }); + if signed { + let overflow = unsafe { entry(i64::MIN as u64, u64::MAX) }; + assert_eq!(overflow, if is_div { i64::MIN as u64 } else { 0 }); } } } @@ -101,7 +134,7 @@ fn division_edges_execute_with_aarch64_semantics() { #[test] fn loop_edges_execute_parallel_parameter_permutations() { let mut f = Function::new(); - let ty = integer(&mut f, false, 64); + let ty = integer(&mut f, 64); let bool_ty = f.types.intern(TypeKind::Bool); let entry = f.append_block(); let head = f.append_block(); @@ -148,8 +181,8 @@ fn loop_edges_execute_parallel_parameter_permutations() { fn signed_byte_load_widens_without_overreading() { let mut f = Function::new(); let ptr = f.types.ptr_global(); - let byte = integer(&mut f, true, 8); - let wide = integer(&mut f, true, 64); + let byte = integer(&mut f, 8); + let wide = integer(&mut f, 64); let b = f.append_block(); let p = f.append_block_param(b, ptr); let value = f.append_inst( @@ -157,14 +190,13 @@ fn signed_byte_load_widens_without_overreading() { byte, Opcode::Load(Load { ptr: p, - volatile: true, + mem: MemFlags { + volatile: true, + ..MemFlags::new() + }, }), ); - let value = f.append_inst( - b, - wide, - Opcode::Convert(volt_ir::function::Convert { value }), - ); + let value = convert(&mut f, b, wide, value, ConvertKind::Sext); f.set_terminator(b, Terminator::Ret(Ret::one(value))); let image = mapped(&f); let entry: unsafe extern "C" fn(*const i8) -> i64 = unsafe { image.entry() }; @@ -172,6 +204,465 @@ fn signed_byte_load_widens_without_overreading() { assert_eq!(unsafe { entry(&byte) }, -97); } +#[cfg(target_arch = "aarch64")] +#[test] +fn big_endian_loads_are_byte_swapped() { + let bytes: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; + for (bits, expected) in [ + (16u16, u16::from_be_bytes([bytes[0], bytes[1]]) as u64), + ( + 32, + u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64, + ), + (64, u64::from_be_bytes(bytes)), + ] { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = integer(&mut f, bits); + let wide = integer(&mut f, 64); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let mem = MemFlags { + endian: Endianness::Big, + ..MemFlags::new() + }; + let value = f.append_load(b, ty, p, mem); + let value = widen(&mut f, b, wide, value, bits, false); + f.set_terminator(b, Terminator::Ret(Ret::one(value))); + let image = mapped(&f); + let entry: unsafe extern "C" fn(*const u8) -> u64 = unsafe { image.entry() }; + assert_eq!(unsafe { entry(bytes.as_ptr()) }, expected, "{bits}-bit"); + } +} + +#[cfg(target_arch = "aarch64")] +#[test] +fn big_endian_stores_are_byte_swapped() { + for bits in [16u16, 32, 64] { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let x = f.append_block_param(b, wide); + let narrow = narrow(&mut f, b, ty, x, bits); + let mem = MemFlags { + endian: Endianness::Big, + ..MemFlags::new() + }; + f.append_store_mem(b, narrow, p, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let image = mapped(&f); + let entry: unsafe extern "C" fn(*mut u8, u64) = unsafe { image.entry() }; + let mut out = [0u8; 8]; + let x = 0x0123_4567_89ab_cdefu64; + unsafe { entry(out.as_mut_ptr(), x) }; + let n = bits as usize / 8; + assert_eq!(out[..n], x.to_be_bytes()[8 - n..], "{bits}-bit"); + assert!(out[n..].iter().all(|b| *b == 0), "{bits}-bit overwrote"); + } +} + +/// `(p: ptr) -> 64-bit` loading a `bits`-wide integer `offset` bytes past `p` with `mem` and +/// widening it (sign- or zero-extending to match `signed`). +fn ordered_load_fn(bits: u16, signed: bool, mem: MemFlags, offset: i64) -> Function { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = integer(&mut f, bits); + let wide = integer(&mut f, 64); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let addr = if offset == 0 { + p + } else { + f.append_arith_imm(b, ptr, BinOp::Add, p, offset) + }; + let value = f.append_load(b, ty, addr, mem); + let value = widen(&mut f, b, wide, value, bits, signed); + f.set_terminator(b, Terminator::Ret(Ret::one(value))); + f +} + +/// `(p: ptr, x: u64)` storing the low `bits` of `x` `offset` bytes past `p` with `mem`. +fn ordered_store_fn(bits: u16, mem: MemFlags, offset: i64) -> Function { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let x = f.append_block_param(b, wide); + let narrow = narrow(&mut f, b, ty, x, bits); + let addr = if offset == 0 { + p + } else { + f.append_arith_imm(b, ptr, BinOp::Add, p, offset) + }; + f.append_store_mem(b, narrow, addr, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + f +} + +fn with_ordering(ordering: Option, bits: u16) -> MemFlags { + MemFlags { + ordering, + align: u32::from(bits / 8), + ..MemFlags::new() + } +} + +const LOAD_ORDERINGS: [Option; 4] = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Acquire), + Some(AtomicOrdering::SeqCst), +]; +const STORE_ORDERINGS: [Option; 4] = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Release), + Some(AtomicOrdering::SeqCst), +]; + +/// Disassembly of every word `compile` emits for `f`. +fn listing(f: &Function) -> Vec { + compile(f) + .unwrap() + .iter() + .map(|&w| crate::aarch64::listing::one(w)) + .collect() +} + +fn mnemonic(line: &str) -> &str { + line.split(' ').next().unwrap() +} + +/// Acquire and sequentially consistent loads select `ldar{b,h,}`, release and sequentially +/// consistent stores `stlr{b,h,}`, each addressed through a bare `[xn]` even when an offset +/// could have been folded; relaxed and plain accesses stay ordinary `ldr`/`str`. +#[test] +fn ordered_accesses_select_load_acquire_and_store_release() { + for (bits, load_acq, store_rel, load_plain, store_plain) in [ + (8u16, "ldarb", "stlrb", "ldrb", "strb"), + (16, "ldarh", "stlrh", "ldrh", "strh"), + (32, "ldar", "stlr", "ldr", "str"), + (64, "ldar", "stlr", "ldr", "str"), + ] { + for offset in [0i64, 8] { + for ordering in LOAD_ORDERINGS { + let lines = listing(&ordered_load_fn( + bits, + false, + with_ordering(ordering, bits), + offset, + )); + let ordered = matches!( + ordering, + Some(AtomicOrdering::Acquire | AtomicOrdering::SeqCst) + ); + let want = if ordered { load_acq } else { load_plain }; + let access: Vec<&String> = lines + .iter() + .filter(|l| mnemonic(l).starts_with("ldr") || mnemonic(l).starts_with("ldar")) + .collect(); + assert_eq!(access.len(), 1, "{bits} {ordering:?} {lines:?}"); + assert_eq!(mnemonic(access[0]), want, "{bits} {ordering:?} {lines:?}"); + if ordered { + assert!( + access[0].ends_with(']') && !access[0].contains(", #"), + "{lines:?}" + ); + assert!(!lines.iter().any(|l| mnemonic(l) == load_plain)); + } + } + for ordering in STORE_ORDERINGS { + let lines = listing(&ordered_store_fn( + bits, + with_ordering(ordering, bits), + offset, + )); + let ordered = matches!( + ordering, + Some(AtomicOrdering::Release | AtomicOrdering::SeqCst) + ); + let want = if ordered { store_rel } else { store_plain }; + let access: Vec<&String> = lines + .iter() + .filter(|l| mnemonic(l).starts_with("str") || mnemonic(l).starts_with("stlr")) + .collect(); + assert_eq!(access.len(), 1, "{bits} {ordering:?} {lines:?}"); + assert_eq!(mnemonic(access[0]), want, "{bits} {ordering:?} {lines:?}"); + if ordered { + assert!( + access[0].ends_with(']') && !access[0].contains(", #"), + "{lines:?}" + ); + } + } + } + } +} + +/// An ordering that has no instruction for the access kind, or a type with no ordered form, +/// must fail closed rather than emit a weaker access. +#[test] +fn unorderable_accesses_fail_closed() { + for ordering in [AtomicOrdering::Release, AtomicOrdering::AcqRel] { + let f = ordered_load_fn(32, false, with_ordering(Some(ordering), 32), 0); + assert!( + matches!(compile(&f), Err(Error::Unsupported)), + "{ordering:?}" + ); + } + for ordering in [AtomicOrdering::Acquire, AtomicOrdering::AcqRel] { + let f = ordered_store_fn(32, with_ordering(Some(ordering), 32), 0); + assert!( + matches!(compile(&f), Err(Error::Unsupported)), + "{ordering:?}" + ); + } + // A 24-bit integer has no ordered form, even when relaxed. + let f = ordered_load_fn( + 24, + false, + with_ordering(Some(AtomicOrdering::Relaxed), 0), + 0, + ); + assert!(matches!(compile(&f), Err(Error::Unsupported))); +} + +#[repr(align(16))] +struct Image([u8; 16]); + +const IMAGE: Image = Image([ + 0x81, 0x92, 0xa3, 0xb4, 0xc5, 0xd6, 0xe7, 0xf8, 0x09, 0x1a, 0x2b, 0x3c, 0x4d, 0x5e, 0x6f, 0x70, +]); + +/// Every width, signedness and ordering loads the same value an ordinary load of that type +/// would (zero- or sign-extended), at offset 0 and at a folded-looking offset 8. +#[cfg(target_arch = "aarch64")] +#[test] +fn ordered_loads_produce_the_plain_load_value() { + let image = IMAGE; + for bits in [8u16, 16, 32, 64] { + for signed in [false, true] { + for offset in [0usize, 8] { + let n = usize::from(bits / 8); + let mut raw = [0u8; 8]; + raw[..n].copy_from_slice(&image.0[offset..offset + n]); + let zero = u64::from_le_bytes(raw); + let expected = if signed && bits < 64 { + (((zero << (64 - bits)) as i64) >> (64 - bits)) as u64 + } else { + zero + }; + for ordering in LOAD_ORDERINGS { + let f = + ordered_load_fn(bits, signed, with_ordering(ordering, bits), offset as i64); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*const u8) -> u64 = unsafe { jitted.entry() }; + assert_eq!( + unsafe { entry(image.0.as_ptr()) }, + expected, + "{bits}-bit signed={signed} offset={offset} {ordering:?}" + ); + } + } + } + } +} + +/// Every width and ordering stores exactly its own bytes. +#[cfg(target_arch = "aarch64")] +#[test] +fn ordered_stores_write_exactly_their_width() { + let x = 0x0123_4567_89ab_cdefu64; + for bits in [8u16, 16, 32, 64] { + for offset in [0usize, 8] { + let n = usize::from(bits / 8); + for ordering in STORE_ORDERINGS { + let f = ordered_store_fn(bits, with_ordering(ordering, bits), offset as i64); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*mut u8, u64) = unsafe { jitted.entry() }; + let mut out = Image([0xee; 16]); + unsafe { entry(out.0.as_mut_ptr(), x) }; + let mut want = [0xeeu8; 16]; + want[offset..offset + n].copy_from_slice(&x.to_le_bytes()[..n]); + assert_eq!(out.0, want, "{bits}-bit offset={offset} {ordering:?}"); + } + } + } +} + +/// Float accesses go through an integer register (legalize reinterprets them) and pointers +/// are 64-bit integers. +#[cfg(target_arch = "aarch64")] +#[test] +fn ordered_float_and_pointer_accesses_execute() { + let mem = |ordering, bytes: u32| MemFlags { + ordering: Some(ordering), + align: bytes, + ..MemFlags::new() + }; + for ordering in [AtomicOrdering::Acquire, AtomicOrdering::SeqCst] { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = f.types.intern(TypeKind::Float(FloatKind::F32)); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let v = f.append_load(b, ty, p, mem(ordering, 4)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + assert!(listing(&f).iter().any(|l| mnemonic(l) == "ldar")); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*const f32) -> f32 = unsafe { jitted.entry() }; + let x = -1.5f32; + assert_eq!(unsafe { entry(&x) }, -1.5); + + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = f.types.intern(TypeKind::Float(FloatKind::F64)); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let v = f.append_load(b, ty, p, mem(ordering, 8)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*const f64) -> f64 = unsafe { jitted.entry() }; + let x = 6.25f64; + assert_eq!(unsafe { entry(&x) }, 6.25); + } + for ordering in [AtomicOrdering::Release, AtomicOrdering::SeqCst] { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let ty = f.types.intern(TypeKind::Float(FloatKind::F64)); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let x = f.append_block_param(b, ty); + f.append_store_mem(b, x, p, mem(ordering, 8)); + f.set_terminator(b, Terminator::Ret(Ret::none())); + assert!(listing(&f).iter().any(|l| mnemonic(l) == "stlr")); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*mut f64, f64) = unsafe { jitted.entry() }; + let mut out = 0.0f64; + unsafe { entry(&mut out, -2.75) }; + assert_eq!(out, -2.75); + } + // Pointer-typed: load a pointer acquire, store it back release elsewhere. + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let b = f.append_block(); + let src = f.append_block_param(b, ptr); + let dst = f.append_block_param(b, ptr); + let v = f.append_load(b, ptr, src, mem(AtomicOrdering::Acquire, 8)); + f.append_store_mem(b, v, dst, mem(AtomicOrdering::Release, 8)); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let lines = listing(&f); + assert!(lines.iter().any(|l| mnemonic(l) == "ldar")); + assert!(lines.iter().any(|l| mnemonic(l) == "stlr")); + let jitted = mapped(&f); + let entry: unsafe extern "C" fn(*const *const u8, *mut *const u8) = unsafe { jitted.entry() }; + let target = 0x1234usize as *const u8; + let mut out = core::ptr::null(); + unsafe { entry(&target, &mut out) }; + assert_eq!(out, target); +} + +/// Message passing: the writer stores data, then a release flag; the reader acquire-loads the +/// flag and then reads the data in the same compiled function. A reader that observes round +/// `i` must observe round `i`'s data. An ack handshake keeps the writer from running ahead, so +/// the expected data is exact on every round. +#[cfg(target_arch = "aarch64")] +#[test] +fn release_acquire_message_passing_between_threads() { + use core::sync::atomic::{AtomicU64, Ordering}; + + // writer(data, flag, value, round): *data = value; release-store *flag = round. + let writer = { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let u64t = integer(&mut f, 64); + let b = f.append_block(); + let data = f.append_block_param(b, ptr); + let flag = f.append_block_param(b, ptr); + let value = f.append_block_param(b, u64t); + let round = f.append_block_param(b, u64t); + f.append_store_mem(b, value, data, with_ordering(None, 64)); + f.append_store_mem( + b, + round, + flag, + with_ordering(Some(AtomicOrdering::Release), 64), + ); + f.set_terminator(b, Terminator::Ret(Ret::none())); + f + }; + // reader(flag, data, out) -> flag: acquire-load the flag, then load the data into *out. + let reader = { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let u64t = integer(&mut f, 64); + let b = f.append_block(); + let flag = f.append_block_param(b, ptr); + let data = f.append_block_param(b, ptr); + let out = f.append_block_param(b, ptr); + let seen = f.append_load( + b, + u64t, + flag, + with_ordering(Some(AtomicOrdering::Acquire), 64), + ); + let value = f.append_load(b, u64t, data, with_ordering(None, 64)); + f.append_store_mem(b, value, out, with_ordering(None, 64)); + f.set_terminator(b, Terminator::Ret(Ret::one(seen))); + f + }; + let writer_image = mapped(&writer); + let reader_image = mapped(&reader); + let write: unsafe extern "C" fn(*mut u64, *mut u64, u64, u64) = unsafe { writer_image.entry() }; + let read: unsafe extern "C" fn(*const u64, *const u64, *mut u64) -> u64 = + unsafe { reader_image.entry() }; + + const ROUNDS: u64 = 1000; + let expected = |round: u64| round.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + let mut data = 0u64; + let mut flag = 0u64; + // The raw addresses cross the thread boundary as integers; each is touched only by the + // compiled code (and the final reads happen after the scope joins). + let (data_addr, flag_addr) = (&raw mut data as usize, &raw mut flag as usize); + let ack = AtomicU64::new(0); + std::thread::scope(|scope| { + scope.spawn(|| { + for round in 1..=ROUNDS { + unsafe { + write( + data_addr as *mut u64, + flag_addr as *mut u64, + expected(round), + round, + ) + }; + while ack.load(Ordering::SeqCst) != round { + std::thread::yield_now(); + } + } + }); + let mut out = 0u64; + for round in 1..=ROUNDS { + loop { + let seen = + unsafe { read(flag_addr as *const u64, data_addr as *const u64, &mut out) }; + if seen == round { + break; + } + assert!(seen < round, "flag ran ahead of the ack handshake"); + std::thread::yield_now(); + } + assert_eq!(out, expected(round), "round {round}"); + ack.store(round, Ordering::SeqCst); + } + }); +} + /// A compiled `u64 -> u64` function and the mapping that keeps it alive. #[cfg(target_arch = "aarch64")] struct NarrowOp { @@ -183,16 +674,18 @@ struct NarrowOp { /// register operand from an `Iconst`, to run both lowerings on the same input. #[cfg(target_arch = "aarch64")] fn narrow_op(signed: bool, bits: u16, op: BinOp, k: i64, immediate: bool) -> NarrowOp { + // The right shift of a signed value is arithmetic. + let op = if signed && op == BinOp::Shr { + BinOp::Sar + } else { + op + }; let mut f = Function::new(); - let wide = integer(&mut f, false, 64); - let ty = integer(&mut f, signed, bits); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); let b = f.append_block(); let x = f.append_block_param(b, wide); - let narrow = f.append_inst( - b, - ty, - Opcode::Convert(volt_ir::function::Convert { value: x }), - ); + let narrow = narrow(&mut f, b, ty, x, bits); let result = if immediate { f.append_arith_imm(b, ty, op, narrow, k) } else { @@ -207,11 +700,7 @@ fn narrow_op(signed: bool, bits: u16, op: BinOp, k: i64, immediate: bool) -> Nar }), ) }; - let back = f.append_inst( - b, - wide, - Opcode::Convert(volt_ir::function::Convert { value: result }), - ); + let back = widen(&mut f, b, wide, result, bits, signed); f.set_terminator(b, Terminator::Ret(Ret::one(back))); let image = mapped(&f); // SAFETY: generated for exactly this AAPCS64 signature; `image` outlives the call. @@ -365,12 +854,614 @@ fn narrow_results_stay_canonical_for_later_uses() { // SAFETY: generated for this signature. unsafe { (f.call)(x) } }; - // u8: a shift that overflows the type wraps to 8 bits. + // i8: a shift that overflows the type wraps to 8 bits. assert_eq!(run(false, 8, BinOp::Shl, 4, 0xff), 0xf0); assert_eq!(run(false, 8, BinOp::Shl, 4, 0x10f), 0xf0); - // u8 & and >> keep the value inside 8 bits. + // i8 & and >> keep the value inside 8 bits. assert_eq!(run(false, 8, BinOp::BitAnd, 0x70, 0xabcd), 0xcd & 0x70); assert_eq!(run(false, 8, BinOp::Shr, 4, 0x1ab), 0xa); - // i8 keeps its sign through an arithmetic shift. + // A signed i8 keeps its sign through an arithmetic shift. assert_eq!(run(true, 8, BinOp::Shr, 2, 0xf0) as u8, 0xfc); } + +/// Signed and unsigned behavior is chosen by the operation: each pair of operations below sees +/// the same bits and must differ exactly where the sign matters. +#[cfg(target_arch = "aarch64")] +mod signedness { + use super::*; + + const WIDTHS: [u16; 4] = [8, 16, 32, 64]; + + fn mask(bits: u16) -> u64 { + if bits == 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + } + } + + /// `v`'s low `bits` as a signed number. + fn sx(bits: u16, v: u64) -> i64 { + let shift = 64 - u32::from(bits); + ((v << shift) as i64) >> shift + } + + /// Values of a `bits`-wide integer (zero-extended) around every sign boundary. + fn samples(bits: u16) -> Vec { + let m = mask(bits); + let max = m >> 1; + let mut v = vec![ + 0, + 1, + 2, + 3, + 7, + 100, + 0x55 & m, + 0xa5 & m, + 127 & m, + 128 & m, + 200 & m, + max, + max + 1, + m - 1, + m, + 0x0123_4567_89ab_cdef & m, + 0xfedc_ba98_7654_3210 & m, + ]; + v.sort_unstable(); + v.dedup(); + v + } + + /// The result of `op` on the `bits`-wide operands `a` and `b` (zero-extended), zero-extended. + fn model(op: BinOp, bits: u16, a: u64, b: u64) -> u64 { + let m = mask(bits); + let (sa, sb) = (i128::from(sx(bits, a)), i128::from(sx(bits, b))); + let r = match op { + BinOp::UDiv => { + if b == 0 { + 0 + } else { + a / b + } + } + BinOp::SDiv => { + if b == 0 { + 0 + } else { + (sa / sb) as u64 + } + } + BinOp::URem => { + if b == 0 { + a + } else { + a % b + } + } + BinOp::SRem => { + if b == 0 { + a + } else { + (sa % sb) as u64 + } + } + BinOp::Shr => a >> b, + BinOp::Sar => (sx(bits, a) >> b) as u64, + BinOp::UMulh => ((u128::from(a) * u128::from(b)) >> bits) as u64, + BinOp::SMulh => ((sa * sb) >> bits) as u64, + _ => unreachable!("{op:?}"), + }; + r & m + } + + /// `(x: u64, y: u64) -> u64`: `op` on the low `bits` of both, zero-extended back. + fn binary_fn(bits: u16, op: BinOp) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let y = f.append_block_param(b, wide); + let nx = narrow(&mut f, b, ty, x, bits); + let ny = narrow(&mut f, b, ty, y, bits); + let r = f.append_inst( + b, + ty, + Opcode::Arith(Arith { + op, + lhs: nx, + rhs: ny, + }), + ); + let r = widen(&mut f, b, wide, r, bits, false); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + f + } + + /// `(x: u64) -> u64`: `x op k` with `k` an `ArithImm` constant. + fn immediate_fn(bits: u16, op: BinOp, k: i64) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let nx = narrow(&mut f, b, ty, x, bits); + let r = f.append_arith_imm(b, ty, op, nx, k); + let r = widen(&mut f, b, wide, r, bits, false); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + f + } + + fn call2(image: &crate::native::JittedFunction, x: u64, y: u64) -> u64 { + // SAFETY: every caller maps a function built for this signature. + let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { image.entry() }; + unsafe { entry(x, y) } + } + + fn call1(image: &crate::native::JittedFunction, x: u64) -> u64 { + // SAFETY: every caller maps a function built for this signature. + let entry: unsafe extern "C" fn(u64) -> u64 = unsafe { image.entry() }; + unsafe { entry(x) } + } + + /// Runs `op` on every pair of samples, with the right operand in a register and (when + /// `immediates`) as a sign-extended `ArithImm`, against `model`. + fn check_binary(ops: &[BinOp], amounts: Option Vec>, immediates: bool) { + for &bits in &WIDTHS { + for &op in ops { + let rights = amounts.map_or_else(|| samples(bits), |f| f(bits)); + let image = mapped(&binary_fn(bits, op)); + for &b in &rights { + let imm = immediates.then(|| mapped(&immediate_fn(bits, op, sx(bits, b)))); + for &a in &samples(bits) { + let want = model(op, bits, a, b); + // The operands carry junk above their width: only the type counts. + let junk = !mask(bits); + let got = call2(&image, a | junk, b | junk); + assert_eq!(got, want, "{op:?} i{bits} {a:#x}, {b:#x}"); + if let Some(imm) = &imm { + let got = call1(imm, a | junk); + assert_eq!(got, want, "{op:?} i{bits} {a:#x}, #{b:#x}"); + } + } + } + } + } + } + + #[test] + fn division_and_remainder_differ_by_operation() { + check_binary( + &[BinOp::UDiv, BinOp::SDiv, BinOp::URem, BinOp::SRem], + None, + true, + ); + // -7 / 2 and -7 % 2 for i8: signed and unsigned disagree, and both are exact. + let (a, b) = (0xf9u64, 2u64); + assert_eq!(model(BinOp::SDiv, 8, a, b), 0xfd); + assert_eq!(model(BinOp::UDiv, 8, a, b), 124); + assert_eq!(model(BinOp::SRem, 8, a, b), 0xff); + assert_eq!(model(BinOp::URem, 8, a, b), 1); + } + + #[test] + fn right_shifts_differ_by_operation() { + fn amounts(bits: u16) -> Vec { + [0, 1, 3, 7, u64::from(bits / 2), u64::from(bits) - 1] + .into_iter() + .filter(|s| *s < u64::from(bits)) + .collect() + } + check_binary(&[BinOp::Shr, BinOp::Sar], Some(amounts), true); + let image = mapped(&binary_fn(8, BinOp::Sar)); + assert_eq!(call2(&image, 0x80, 7), 0xff); + let image = mapped(&binary_fn(8, BinOp::Shr)); + assert_eq!(call2(&image, 0x80, 7), 1); + } + + #[test] + fn high_products_differ_by_operation() { + check_binary(&[BinOp::UMulh, BinOp::SMulh], None, true); + // i64: (-1) * (-1) has high half 0 signed, but 0xffff_ffff_ffff_fffe unsigned. + let image = mapped(&binary_fn(64, BinOp::SMulh)); + assert_eq!(call2(&image, u64::MAX, u64::MAX), 0); + let image = mapped(&binary_fn(64, BinOp::UMulh)); + assert_eq!(call2(&image, u64::MAX, u64::MAX), u64::MAX - 1); + } + + /// `(x, y) -> (x cmp y) as u64` on the low `bits`; branching on the compare when `branch`, + /// so the compare-and-branch fusion sees it. + fn compare_fn(bits: u16, op: CmpOp, branch: bool) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let bool_ty = f.types.intern(TypeKind::Bool); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let y = f.append_block_param(b, wide); + let nx = narrow(&mut f, b, ty, x, bits); + let ny = narrow(&mut f, b, ty, y, bits); + let cond = f.append_inst( + b, + bool_ty, + Opcode::Icmp(Compare { + op, + lhs: nx, + rhs: ny, + }), + ); + if branch { + let yes = f.append_block(); + let no = f.append_block(); + f.append_if(b, cond, EdgeDesc::bare(yes), EdgeDesc::bare(no)); + let one = f.append_inst(yes, wide, Opcode::Iconst(1)); + f.set_terminator(yes, Terminator::Ret(Ret::one(one))); + let zero = f.append_inst(no, wide, Opcode::Iconst(0)); + f.set_terminator(no, Terminator::Ret(Ret::one(zero))); + } else { + let r = convert(&mut f, b, wide, cond, ConvertKind::Zext); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + } + f + } + + fn compare_model(op: CmpOp, bits: u16, a: u64, b: u64) -> bool { + let (sa, sb) = (sx(bits, a), sx(bits, b)); + match op { + CmpOp::Eq => a == b, + CmpOp::Ne => a != b, + CmpOp::Slt => sa < sb, + CmpOp::Sle => sa <= sb, + CmpOp::Sgt => sa > sb, + CmpOp::Sge => sa >= sb, + CmpOp::Ult => a < b, + CmpOp::Ule => a <= b, + CmpOp::Ugt => a > b, + CmpOp::Uge => a >= b, + _ => unreachable!("{op:?}"), + } + } + + #[test] + fn integer_compares_differ_by_operation() { + let ops = [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Slt, + CmpOp::Sle, + CmpOp::Sgt, + CmpOp::Sge, + CmpOp::Ult, + CmpOp::Ule, + CmpOp::Ugt, + CmpOp::Uge, + ]; + for bits in WIDTHS { + for op in ops { + for branch in [false, true] { + let image = mapped(&compare_fn(bits, op, branch)); + let junk = !mask(bits); + for &a in &samples(bits) { + for &b in &samples(bits) { + let got = call2(&image, a | junk, b | junk); + let want = u64::from(compare_model(op, bits, a, b)); + assert_eq!( + got, want, + "{op:?} i{bits} {a:#x}, {b:#x} (branch: {branch})" + ); + } + } + } + } + } + } + + #[test] + fn pointer_compares_are_unsigned() { + let mut f = Function::new(); + let ptr = f.types.ptr_global(); + let wide = integer(&mut f, 64); + let bool_ty = f.types.intern(TypeKind::Bool); + let b = f.append_block(); + let p = f.append_block_param(b, ptr); + let q = f.append_block_param(b, ptr); + let cond = f.append_inst( + b, + bool_ty, + Opcode::Icmp(Compare { + op: CmpOp::Ult, + lhs: p, + rhs: q, + }), + ); + let r = convert(&mut f, b, wide, cond, ConvertKind::Zext); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + let image = mapped(&f); + assert_eq!(call2(&image, 1, u64::MAX), 1); + assert_eq!(call2(&image, u64::MAX, 1), 0); + } + + /// `(x: u64) -> u64`: `x` truncated to `from` bits, converted to `to` bits with `kind`, and + /// zero-extended to 64 bits. + fn int_convert_fn(kind: ConvertKind, from: u16, to: u16) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let a = integer(&mut f, from); + let c = integer(&mut f, to); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let x = narrow(&mut f, b, a, x, from); + let y = convert(&mut f, b, c, x, kind); + let y = widen(&mut f, b, wide, y, to, false); + f.set_terminator(b, Terminator::Ret(Ret::one(y))); + f + } + + #[test] + fn extensions_and_truncations_differ_by_kind() { + for from in WIDTHS { + for to in WIDTHS { + if from < to { + for kind in [ConvertKind::Zext, ConvertKind::Sext] { + let image = mapped(&int_convert_fn(kind, from, to)); + for &x in &samples(from) { + let want = if kind == ConvertKind::Sext { + sx(from, x) as u64 & mask(to) + } else { + x + }; + let got = call1(&image, x | !mask(from)); + assert_eq!(got, want, "{kind:?} i{from} -> i{to} {x:#x}"); + } + } + } else if from > to { + let image = mapped(&int_convert_fn(ConvertKind::Trunc, from, to)); + for &x in &samples(from) { + let got = call1(&image, x); + assert_eq!(got, x & mask(to), "Trunc i{from} -> i{to} {x:#x}"); + } + } + } + } + } + + const FLOAT_INPUTS: [f64; 23] = [ + 0.0, + 1.5, + -1.5, + 127.0, + 128.0, + 200.0, + -129.0, + 255.9, + 256.0, + 40000.0, + 70000.0, + -70000.0, + 2147483647.0, + 2147483648.0, + 4294967295.0, + 4294967296.0, + -1.0e10, + 1.0e10, + 9.3e18, + -9.3e18, + 1.0e30, + -1.0e30, + f64::NAN, + ]; + + fn float_type(f: &mut Function, double: bool) -> Type { + f.types.intern(TypeKind::Float(if double { + FloatKind::F64 + } else { + FloatKind::F32 + })) + } + + /// `(x: u64) -> float`: the low `bits` of `x`, converted with `kind`. + fn int_to_float_fn(kind: ConvertKind, bits: u16, double: bool) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let fty = float_type(&mut f, double); + let b = f.append_block(); + let x = f.append_block_param(b, wide); + let x = narrow(&mut f, b, ty, x, bits); + let y = convert(&mut f, b, fty, x, kind); + f.set_terminator(b, Terminator::Ret(Ret::one(y))); + f + } + + /// `(x: float) -> u64`: `x` converted to `bits` with `kind`, zero-extended. + fn float_to_int_fn(kind: ConvertKind, bits: u16, double: bool) -> Function { + let mut f = Function::new(); + let wide = integer(&mut f, 64); + let ty = integer(&mut f, bits); + let fty = float_type(&mut f, double); + let b = f.append_block(); + let x = f.append_block_param(b, fty); + let y = convert(&mut f, b, ty, x, kind); + let y = widen(&mut f, b, wide, y, bits, false); + f.set_terminator(b, Terminator::Ret(Ret::one(y))); + f + } + + #[test] + fn integer_to_float_differs_by_kind() { + let inputs = { + let mut v = Vec::new(); + for bits in WIDTHS { + v.extend(samples(bits)); + } + v + }; + for bits in WIDTHS { + for kind in [ConvertKind::SiToFp, ConvertKind::UiToFp] { + let signed = kind == ConvertKind::SiToFp; + let single = mapped(&int_to_float_fn(kind, bits, false)); + let double = mapped(&int_to_float_fn(kind, bits, true)); + // SAFETY: both were built for these signatures. + let single: unsafe extern "C" fn(u64) -> f32 = unsafe { single.entry() }; + let double: unsafe extern "C" fn(u64) -> f64 = unsafe { double.entry() }; + for &x in &inputs { + let v = x & mask(bits); + let (want32, want64) = if signed { + (sx(bits, v) as f32, sx(bits, v) as f64) + } else { + (v as f32, v as f64) + }; + // Garbage above the type must not matter. + let arg = x | !mask(bits); + let arg = if bits == 64 { x } else { arg }; + let (got32, got64) = unsafe { (single(arg), double(arg)) }; + assert_eq!(got32, want32, "{kind:?} i{bits} {x:#x} -> f32"); + assert_eq!(got64, want64, "{kind:?} i{bits} {x:#x} -> f64"); + } + } + } + } + + #[test] + fn float_to_integer_differs_by_kind() { + for bits in WIDTHS { + for kind in [ConvertKind::FpToSi, ConvertKind::FpToUi] { + let signed = kind == ConvertKind::FpToSi; + let single = mapped(&float_to_int_fn(kind, bits, false)); + let double = mapped(&float_to_int_fn(kind, bits, true)); + // SAFETY: both were built for these signatures. + let single: unsafe extern "C" fn(f32) -> u64 = unsafe { single.entry() }; + let double: unsafe extern "C" fn(f64) -> u64 = unsafe { double.entry() }; + let want = |narrow32: u64, wide64: u64| { + // A 64-bit destination saturates at 64 bits; every narrower one keeps the + // low bits of the 32-bit saturating conversion. + if bits == 64 { + wide64 + } else { + narrow32 & mask(bits) + } + }; + for &x in &FLOAT_INPUTS { + let (w32, w64) = if signed { + ( + u64::from((x as f32 as i32) as u32), + (x as f32 as i64) as u64, + ) + } else { + (u64::from(x as f32 as u32), x as f32 as u64) + }; + assert_eq!( + unsafe { single(x as f32) }, + want(w32, w64), + "{kind:?} f32 {x} -> i{bits}" + ); + let (w32, w64) = if signed { + (u64::from((x as i32) as u32), (x as i64) as u64) + } else { + (u64::from(x as u32), x as u64) + }; + assert_eq!( + unsafe { double(x) }, + want(w32, w64), + "{kind:?} f64 {x} -> i{bits}" + ); + } + } + } + } + + #[test] + fn float_resize_extends_and_truncates() { + let build = |from_double: bool| { + let mut f = Function::new(); + let a = float_type(&mut f, from_double); + let c = float_type(&mut f, !from_double); + let b = f.append_block(); + let x = f.append_block_param(b, a); + let y = convert(&mut f, b, c, x, ConvertKind::FpResize); + f.set_terminator(b, Terminator::Ret(Ret::one(y))); + mapped(&f) + }; + let widen = build(false); + let shrink = build(true); + // SAFETY: built for these signatures. + let widen: unsafe extern "C" fn(f32) -> f64 = unsafe { widen.entry() }; + let shrink: unsafe extern "C" fn(f64) -> f32 = unsafe { shrink.entry() }; + for x in [0.0f64, -1.5, 1.0e10, 3.4e38, 1.0e300, f64::INFINITY] { + assert_eq!(unsafe { shrink(x) }, x as f32); + assert_eq!(unsafe { widen(x as f32) }, f64::from(x as f32)); + } + } + + /// Integer vector lanes are held zero-extended; widening them is up to the program. + #[test] + fn vector_lanes_extend_by_explicit_conversion() { + for (kind, want) in [ + (ConvertKind::Sext, 0xffff_ffff_8000_0000u64), + (ConvertKind::Zext, 0x8000_0000), + ] { + let mut f = Function::new(); + let i32t = integer(&mut f, 32); + let wide = integer(&mut f, 64); + let vt = f + .types + .intern(TypeKind::Vector(VectorDesc { len: 4, elem: i32t })); + let b = f.append_block(); + let p: Vec = (0..4).map(|_| f.append_block_param(b, i32t)).collect(); + let v = f.append_struct_new(b, vt, &p); + let lane = f.append_inst( + b, + i32t, + Opcode::Extract(Extract { + aggregate: v, + index: 2, + }), + ); + let r = convert(&mut f, b, wide, lane, kind); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + let image = mapped(&f); + // SAFETY: built for this signature. + let entry: unsafe extern "C" fn(u32, u32, u32, u32) -> u64 = unsafe { image.entry() }; + assert_eq!(unsafe { entry(1, 2, 0x8000_0000, 4) }, want, "{kind:?}"); + } + } + + #[test] + fn dot_selects_signed_or_unsigned_by_flag() { + for signed in [false, true] { + let mut f = Function::new(); + let i8t = integer(&mut f, 8); + let i32t = integer(&mut f, 32); + let bytes = f + .types + .intern(TypeKind::Vector(VectorDesc { len: 16, elem: i8t })); + let acc_ty = f + .types + .intern(TypeKind::Vector(VectorDesc { len: 4, elem: i32t })); + let b = f.append_block(); + let acc = f.append_block_param(b, acc_ty); + let x = f.append_block_param(b, bytes); + let y = f.append_block_param(b, bytes); + let r = f.append_inst( + b, + acc_ty, + Opcode::Dot(Dot { + acc, + a: x, + b: y, + signed, + }), + ); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + let code = compile(&f).unwrap(); + let v = crate::aarch64::encode::Reg::from_u5(0); + let sdot = crate::aarch64::encode::sdot(v, v, v); + let udot = crate::aarch64::encode::udot(v, v, v); + let find = |word: u32| code.iter().any(|w| w & 0xffe0_fc00 == word & 0xffe0_fc00); + assert_eq!(find(sdot), signed, "signed={signed}"); + assert_eq!(find(udot), !signed, "signed={signed}"); + } + } +} diff --git a/src/target/src/aarch64/jit.rs b/src/target/src/aarch64/jit.rs index c2359ad..2b61064 100644 --- a/src/target/src/aarch64/jit.rs +++ b/src/target/src/aarch64/jit.rs @@ -41,3 +41,88 @@ impl Compiled { unsafe { self.mapping.entry_at(offset) } } } + +#[cfg(all(test, target_arch = "aarch64"))] +mod tests { + use super::*; + use crate::aarch64::{isel::ModelCaps, link::compile_module}; + use volt_ir::function::{ + ArithImm, BinOp, Function, GlobalAddr, MemFlags, Opcode, Ret, Terminator, + }; + use volt_ir::module::{DataReloc, Module}; + + #[test] + fn linked_module_maps_calls_globals_and_data_relocations() { + // inc(x) = x + 1 + let mut inc = Function::new(); + let u64_t = inc.types.parse_type("i64").unwrap(); + let b = inc.append_block(); + let x = inc.append_block_param(b, u64_t); + let r = inc.append_inst( + b, + u64_t, + Opcode::ArithImm(ArithImm { + op: BinOp::Add, + lhs: x, + imm: 1, + }), + ); + inc.set_terminator(b, Terminator::Ret(Ret::one(r))); + // twice(x) = inc(inc(x)) + let mut twice = Function::new(); + let u64_t = twice.types.parse_type("i64").unwrap(); + let b = twice.append_block(); + let x = twice.append_block_param(b, u64_t); + let once = twice.append_call(b, u64_t, "inc", &[x]); + let r = twice.append_call(b, u64_t, "inc", &[once]); + twice.set_terminator(b, Terminator::Ret(Ret::one(r))); + // seed() = *seed_value + let mut seed = Function::new(); + let u64_t = seed.types.parse_type("i64").unwrap(); + let ptr_t = seed.types.parse_type("ptr").unwrap(); + let b = seed.append_block(); + let symbol = seed.intern_symbol("seed_value"); + let addr = seed.append_inst( + b, + ptr_t, + Opcode::GlobalAddr(GlobalAddr { + symbol, + via_got: false, + }), + ); + let v = seed.append_load(b, u64_t, addr, MemFlags::new()); + seed.set_terminator(b, Terminator::Ret(Ret::one(v))); + + let mut module = Module::new(); + module.add_function("twice", twice); + module.add_function("inc", inc); + module.add_function("seed", seed); + module.add_writable("seed_value", 40u64.to_le_bytes().to_vec()); + module.add_data_relocs( + "ptrs", + vec![0; 8], + vec![DataReloc { + offset: 0, + symbol: "inc".into(), + }], + ); + let linked = compile_module(&module, &ModelCaps::default()).unwrap(); + let compiled = Compiled::new(linked).unwrap(); + // SAFETY: signatures match the IR above and the host is AArch64. + unsafe { + let twice: extern "C" fn(u64) -> u64 = compiled.entry("twice").unwrap(); + assert_eq!(twice(5), 7); + let seed: extern "C" fn() -> u64 = compiled.entry("seed").unwrap(); + assert_eq!(seed(), 40); + let inc: extern "C" fn(u64) -> u64 = compiled.entry("inc").unwrap(); + let slot = compiled + .data_addr("ptrs") + .unwrap() + .cast::() + .read_unaligned(); + assert_eq!(slot, inc as usize as u64); + } + assert_eq!(compiled.offset_of("twice"), Some(0)); + assert!(!compiled.code().is_empty()); + } +} diff --git a/src/target/src/aarch64/link.rs b/src/target/src/aarch64/link.rs index 92354c9..4e13c5d 100644 --- a/src/target/src/aarch64/link.rs +++ b/src/target/src/aarch64/link.rs @@ -1,16 +1,18 @@ -//! AArch64 static/JIT linker: concatenate pre-compiled functions, resolve -//! intra-module calls, lay out data globals, and carry `global_addr` -//! relocations forward. +//! AArch64 static/JIT linker: compile a [`volt_ir::module::Module`], +//! concatenate its functions, resolve intra-module calls, lay out data +//! globals, and carry `global_addr` relocations forward. //! -//! [`Module`] takes pre-compiled [`Vec`] function bodies. Compile a -//! `volt_ir::function::Function` through [`super::isel::compile_function`] -//! first, retaining relocations for object emission or native module mapping. +//! Each function goes through [`super::isel::compile_function`]; the +//! relocations it retains drive linking here and, for object emission or native +//! module mapping, the unresolved ones are carried in [`Linked`]. -use alloc::string::{String, ToString as _}; +use alloc::string::String; use alloc::vec::Vec; +use volt_ir::module::{DataKind, DataReloc, Module}; + use super::encode; -use super::isel::{Reloc, RelocKind}; +use super::isel::{self, ModelCaps, Reloc, RelocKind}; /// Linker failure. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -36,112 +38,6 @@ impl core::fmt::Display for Error { impl core::error::Error for Error {} -/// The section a data global lands in: read-only, writable, or zero-initialized. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum DataKind { - Rodata, - Data, - Bss, -} - -/// An internal relocation within a data object. At byte offset `off` within -/// the object, the linker must write a pointer-sized absolute runtime address -/// of `symbol` once the module is mapped. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct DataReloc { - pub off: usize, - pub symbol: String, -} - -/// A named data global placed in the linked output. For `.bss`, `bytes` is -/// empty and `size` gives the zero-initialized length; otherwise -/// `size == bytes.len` as `u64`. -#[derive(Clone, Debug)] -pub struct Data { - pub name: String, - pub bytes: Vec, - pub kind: DataKind, - pub size: u64, - pub relocs: Vec, -} - -/// A set of named functions and data globals to link together. The first added -/// function is the entry point at offset 0 of the linked image. -#[derive(Clone, Debug, Default)] -pub struct Module { - pub functions: Vec<(String, Vec)>, - pub data: Vec, -} - -impl Module { - pub fn new() -> Module { - Module::default() - } - - pub fn add_function(&mut self, name: &str, code: Vec) { - self.functions.push((name.to_string(), code)); - } - - /// Add a named read-only data blob (a global constant) into `.rodata`. - pub fn add_data(&mut self, name: &str, bytes: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.to_string(), - bytes, - kind: DataKind::Rodata, - size, - relocs: Vec::new(), - }); - } - - /// Add a named read-only data blob carrying internal relocations. - pub fn add_data_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.to_string(), - bytes, - kind: DataKind::Rodata, - size, - relocs, - }); - } - - /// Add a named writable data global into `.data`. - pub fn add_writable(&mut self, name: &str, bytes: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.to_string(), - bytes, - kind: DataKind::Data, - size, - relocs: Vec::new(), - }); - } - - /// Add a named writable data global carrying internal relocations. - pub fn add_writable_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.to_string(), - bytes, - kind: DataKind::Data, - size, - relocs, - }); - } - - /// Add a named zero-initialized data global of `size` bytes into `.bss`. - pub fn add_bss(&mut self, name: &str, size: u64) { - self.data.push(Data { - name: name.to_string(), - bytes: Vec::new(), - kind: DataKind::Bss, - size, - relocs: Vec::new(), - }); - } -} - /// A function's byte offset within the linked image. #[derive(Clone, Debug, Eq, PartialEq)] pub struct Symbol { @@ -157,6 +53,8 @@ pub struct DataSym { pub kind: DataKind, pub off: usize, pub size: usize, + /// The global's requested alignment (`0` = natural), as in [`volt_ir::module::Global::align`]. + pub align: u32, pub bytes: Vec, pub relocs: Vec, } @@ -182,14 +80,6 @@ impl Linked { } } -/// A data object's natural alignment: the next power of two up to `size`, -/// capped at 8. Covers scalars and pointers up to a 64-bit word. -pub fn align_of_data(size: u64) -> usize { - let size = size.max(1); - let pow = size.next_power_of_two(); - pow.min(8) as usize -} - fn align_forward(off: usize, align: usize) -> usize { let mask = align - 1; (off + mask) & !mask @@ -228,36 +118,42 @@ pub fn apply_global_reloc(code: &mut [u32], r: &Reloc, site_addr: usize, target_ } } -/// Concatenate the module's functions, resolve each intra-module `call` -/// relocation to a PC-relative `bl`, lay out data globals into per-section -/// `DataSym`s, and carry every `global_addr` relocation forward unresolved. +/// Compile every function of `module` with `caps`, concatenate the code, +/// resolve each intra-module `call` relocation to a PC-relative `bl`, lay out +/// data globals into per-section `DataSym`s, and carry every `global_addr` +/// relocation forward unresolved. /// -/// `relocs` are supplied alongside the module: `relocs[i]` holds the isel -/// relocations for `module.functions[i]`. -pub fn compile_module(module: &Module, relocs: &[Vec]) -> Result { - let n = module.functions.len(); +/// The first function sits at offset 0 of the image. Symbols the module does +/// not define are an error for calls ([`Error::UndefinedSymbol`]) but are left +/// unresolved for `global_addr` relocations and data relocations. +pub fn compile_module(module: &Module, caps: &ModelCaps) -> Result { + let compiled = module + .functions + .iter() + .map(|def| isel::compile_function(&def.function, caps).map_err(|_| Error::Unsupported)) + .collect::, _>>()?; + let n = compiled.len(); let mut word_off = Vec::with_capacity(n); let mut total = 0usize; - for (_, code) in module.functions.iter() { + for part in &compiled { word_off.push(total); - total += code.len(); + total += part.code.len(); } let mut code: Vec = Vec::with_capacity(total); - for (_, words) in module.functions.iter() { - code.extend_from_slice(words); + for part in &compiled { + code.extend_from_slice(&part.code); } let mut global_relocs: Vec = Vec::new(); - for (i, rs) in relocs.iter().enumerate() { - for r in rs { + for (i, part) in compiled.iter().enumerate() { + for r in &part.relocs { let at = word_off[i] + r.offset; match r.kind { RelocKind::Call => { let target_word = module .functions .iter() - .enumerate() - .find(|(_, (name, _))| *name == r.symbol) - .map(|(j, _)| word_off[j]) + .position(|def| def.name == r.symbol) + .map(|j| word_off[j]) .ok_or(Error::UndefinedSymbol)?; let byte_off = (target_word as i64 - at as i64) * 4; if !(-(128 << 20)..=(128 << 20) - 4).contains(&byte_off) { @@ -280,54 +176,32 @@ pub fn compile_module(module: &Module, relocs: &[Vec]) -> Result = Vec::with_capacity(module.data.len()); + let mut data_syms: Vec = Vec::with_capacity(module.globals.len()); let (mut rodata_off, mut data_off, mut bss_off) = (0usize, 0usize, 0usize); - for d in module.data.iter() { - let a = align_of_data(d.size); - let size = d.size as usize; - match d.kind { - DataKind::Rodata => { - rodata_off = align_forward(rodata_off, a); - data_syms.push(DataSym { - name: d.name.clone(), - kind: d.kind, - off: rodata_off, - size, - bytes: d.bytes.clone(), - relocs: d.relocs.clone(), - }); - rodata_off += size; - } - DataKind::Data => { - data_off = align_forward(data_off, a); - data_syms.push(DataSym { - name: d.name.clone(), - kind: d.kind, - off: data_off, - size, - bytes: d.bytes.clone(), - relocs: d.relocs.clone(), - }); - data_off += size; - } - DataKind::Bss => { - bss_off = align_forward(bss_off, a); - data_syms.push(DataSym { - name: d.name.clone(), - kind: d.kind, - off: bss_off, - size, - bytes: Vec::new(), - relocs: Vec::new(), - }); - bss_off += size; - } - } + for g in module.globals.iter() { + let a = g.effective_align(); + let size = g.size as usize; + let (cursor, bytes, relocs) = match g.kind { + DataKind::Rodata => (&mut rodata_off, g.bytes.clone(), g.relocs.clone()), + DataKind::Data => (&mut data_off, g.bytes.clone(), g.relocs.clone()), + DataKind::Bss => (&mut bss_off, Vec::new(), Vec::new()), + }; + *cursor = align_forward(*cursor, a); + data_syms.push(DataSym { + name: g.name.clone(), + kind: g.kind, + off: *cursor, + size, + align: g.align, + bytes, + relocs, + }); + *cursor += size; } Ok(Linked { code, @@ -339,16 +213,121 @@ pub fn compile_module(module: &Module, relocs: &[Vec]) -> Result Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let v = f.append_inst(block, ty, Opcode::Iconst(value)); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + f + } + + fn function_calling(callee: &str) -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let symbol = f.intern_symbol(callee); + let args = f.intern_values(&[]); + let result = f.append_inst( + block, + ty, + Opcode::Call(Call { + symbol, + args, + is_variadic: false, + num_fixed: 0, + ret_dest: None, + ret_regs: 0, + ret_pieces: [RetPiece::default(); 4], + sret: false, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(result))); + f + } + + fn function_taking_address_of(symbol: &str) -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("ptr").unwrap(); + let block = f.append_block(); + let symbol = f.intern_symbol(symbol); + let v = f.append_inst( + block, + ty, + Opcode::GlobalAddr(GlobalAddr { + symbol, + via_got: false, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + f + } + + #[test] + fn globals_are_laid_out_per_section_with_alignment() { + let mut m = Module::new(); + m.add_data("ro1", vec![1, 2, 3]); + m.add_data("ro2", vec![0; 8]); + m.add_writable("w1", vec![9]); + m.add_writable("w2", vec![0; 4]); + m.add_bss("b1", 3); + m.add_bss("b2", 64); + m.globals[5].align = 64; + let linked = compile_module(&m, &ModelCaps::default()).unwrap(); + let place = |name: &str| { + let d = linked.data.iter().find(|d| d.name == name).unwrap(); + (d.kind, d.off, d.size) + }; + assert_eq!(place("ro1"), (DataKind::Rodata, 0, 3)); + // `ro2` is 8 bytes: aligned to 8 after the 3-byte `ro1`. + assert_eq!(place("ro2"), (DataKind::Rodata, 8, 8)); + assert_eq!(place("w1"), (DataKind::Data, 0, 1)); + assert_eq!(place("w2"), (DataKind::Data, 4, 4)); + assert_eq!(place("b1"), (DataKind::Bss, 0, 3)); + // An explicit alignment overrides the natural (<= 8) one. + assert_eq!(place("b2"), (DataKind::Bss, 64, 64)); + assert!( + linked + .data + .iter() + .find(|d| d.name == "b2") + .unwrap() + .bytes + .is_empty() + ); + } + #[test] - fn data_alignment_covers_scalars() { - assert_eq!(align_of_data(1), 1); - assert_eq!(align_of_data(3), 4); - assert_eq!(align_of_data(8), 8); - assert_eq!(align_of_data(100), 8); + fn global_addr_relocations_carry_forward_unresolved() { + let mut m = Module::new(); + m.add_function("first", function_returning(0)); + m.add_function("addr", function_taking_address_of("table")); + m.add_writable_relocs( + "table", + vec![0; 8], + vec![DataReloc { + offset: 0, + symbol: "first".to_string(), + }], + ); + let linked = compile_module(&m, &ModelCaps::default()).unwrap(); + let start_word = linked.address_of("addr").unwrap() / 4; + let kinds: Vec<_> = linked + .relocs + .iter() + .map(|r| (r.kind, r.symbol.as_str())) + .collect(); + assert!(kinds.contains(&(RelocKind::AdrpPg, "table"))); + assert!(kinds.contains(&(RelocKind::AddPgOff, "table"))); + // Carried-forward offsets are word indices into the whole image. + assert!(linked.relocs.iter().all(|r| r.offset >= start_word)); + assert_eq!(linked.data[0].relocs[0].symbol, "first"); } #[test] @@ -369,15 +348,30 @@ mod tests { #[test] fn compile_module_resolves_call() { let mut m = Module::new(); - m.add_function("callee", vec![0xD65F_03C0]); - m.add_function("caller", vec![encode::bl(0)]); - let relocs = vec![Vec::new(), vec![Reloc::call(0, "callee")]]; - let linked = compile_module(&m, &relocs).unwrap(); + m.add_function("callee", function_returning(42)); + m.add_function("caller", function_calling("callee")); + let linked = compile_module(&m, &ModelCaps::default()).unwrap(); + let caller = linked.address_of("caller").unwrap(); assert_eq!(linked.address_of("callee"), Some(0)); - assert_eq!(linked.address_of("caller"), Some(4)); - // caller word 1 branches back one word: byte offset -4. - assert_eq!(linked.code[1], encode::bl(-4)); + assert!(caller > 0 && caller % 4 == 0); assert!(linked.relocs.is_empty()); + // The caller contains exactly one `bl`, which branches back to word 0. + let bls: Vec = (caller / 4..linked.code.len()) + .filter(|&i| linked.code[i] >> 26 == 0b100101) + .collect(); + assert_eq!(bls.len(), 1); + let at = bls[0]; + assert_eq!(linked.code[at], encode::bl(-((at * 4) as i32))); + } + + #[test] + fn undefined_call_target_is_an_error() { + let mut m = Module::new(); + m.add_function("caller", function_calling("nowhere")); + assert_eq!( + compile_module(&m, &ModelCaps::default()).unwrap_err(), + Error::UndefinedSymbol + ); } #[test] diff --git a/src/target/src/aarch64/listing.rs b/src/target/src/aarch64/listing.rs new file mode 100644 index 0000000..c750869 --- /dev/null +++ b/src/target/src/aarch64/listing.rs @@ -0,0 +1,272 @@ +//! Assembly listings of AArch64 code buffers, for debugging code generation. +//! +//! The text of each instruction comes from `volt_isa_aarch64::format`; this module +//! only lays words out with addresses, labels, source-line markers and symbol +//! annotations. A word the decoder does not recognise prints as `.word 0x`. + +use alloc::string::{String, ToString as _}; +use core::fmt::Write as _; +use volt_isa_aarch64::flow::flow; +use volt_isa_aarch64::format::{self, Symbols}; + +/// A source-line-table row: the byte offset where a source line's code begins. +pub struct SourceLine { + pub offset: u32, + pub line: u32, +} + +/// A function's location in a linked image: its name and its word offset. +pub struct Sym<'a> { + pub name: &'a str, + pub word: usize, +} + +/// A defined symbol by absolute address (a function or data object), for +/// resolving branch and `adrp` targets when listing an ELF image. +pub struct AddrSym<'a> { + pub name: &'a str, + pub addr: u64, + pub size: u64, + pub is_func: bool, +} + +/// A decoded source-line row keyed by absolute address (from a `.debug_line` +/// program), for objdump `-S`-style annotation of a real ELF. +pub struct AddrLine { + pub addr: u64, + pub line: u32, +} + +/// The symbol containing an address: the greatest-addressed symbol at or before it. +struct AddrSyms<'a>(&'a [AddrSym<'a>]); + +impl Symbols for AddrSyms<'_> { + fn lookup(&self, addr: u64) -> Option<(&str, u64)> { + let mut best: Option<&AddrSym<'_>> = None; + for s in self.0 { + if s.addr <= addr && best.is_none_or(|b| s.addr > b.addr) { + best = Some(s); + } + } + best.map(|s| (s.name, addr - s.addr)) + } +} + +/// Render one instruction word as `mnemonic operands`. +pub fn one(word: u32) -> String { + format::word(word).to_string() +} + +/// Render a whole instruction stream (one word per line, `addr: hex text`). +pub fn format(code: &[u32]) -> String { + let mut out = String::new(); + for (i, &word) in code.iter().enumerate() { + let _ = writeln!(out, "{:04x}: {word:08x} {}", i * 4, format::word(word)); + } + out +} + +/// Render a listing with source-line markers interleaved (objdump `-S` +/// style): a `; line N` header precedes the instructions belonging to that +/// source line. `lines` must be sorted by offset (as isel produces). +pub fn format_with_lines(code: &[u32], lines: &[SourceLine]) -> String { + let mut out = String::new(); + let mut next = 0; + for (i, &word) in code.iter().enumerate() { + let offset = (i * 4) as u32; + while next < lines.len() && lines[next].offset == offset { + let _ = writeln!(out, "; line {}", lines[next].line); + next += 1; + } + let _ = writeln!(out, "{offset:04x}: {word:08x} {}", format::word(word)); + } + out +} + +/// Render a whole linked module: each function gets a `name:` label at its +/// offset, and every PC-relative branch is annotated with the function it lands +/// in (`` or ``), so a linked image reads like a symbolized +/// listing. +pub fn format_module(code: &[u32], syms: &[Sym<'_>]) -> String { + let mut out = String::new(); + for (i, &word) in code.iter().enumerate() { + for s in syms.iter().filter(|s| s.word == i) { + if i != 0 { + out.push('\n'); + } + let _ = writeln!(out, "{}:", s.name); + } + let _ = write!(out, "{:04x}: {word:08x} {}", i * 4, format::word(word)); + let target = flow(word, (i * 4) as u64) + .target() + .filter(|&t| (t as i64) >= 0) + .map(|t| (t / 4) as usize); + if let Some(target) = target + && let Some(s) = syms + .iter() + .filter(|s| s.word <= target) + .max_by_key(|s| s.word) + { + if target == s.word { + let _ = write!(out, " <{}>", s.name); + } else { + let _ = write!(out, " <{}+0x{:x}>", s.name, (target - s.word) * 4); + } + } + out.push('\n'); + } + out +} + +/// Render an ELF code section as an objdump-style listing: absolute +/// addresses, function labels, and branch/`adrp` targets resolved to +/// `0x `. `base` is the section's load address; `syms` are all +/// defined symbols (functions and data). +pub fn format_elf(code: &[u32], base: u64, syms: &[AddrSym<'_>]) -> String { + format_elf_with_lines(code, base, syms, &[]) +} + +/// Like [`format_elf`], but interleaves `; line N` markers from `lines` +/// (sorted by address) before the instruction at each address, so a listing +/// of a `-g` object reads with its source lines. +pub fn format_elf_with_lines( + code: &[u32], + base: u64, + syms: &[AddrSym<'_>], + lines: &[AddrLine], +) -> String { + let symbols = AddrSyms(syms); + let mut out = String::new(); + let mut next = 0; + for (i, &word) in code.iter().enumerate() { + let addr = base + i as u64 * 4; + for s in syms.iter().filter(|s| s.is_func && s.addr == addr) { + if i != 0 { + out.push('\n'); + } + let _ = writeln!(out, "{addr:016x} <{}>:", s.name); + } + while next < lines.len() && lines[next].addr < addr { + next += 1; // skip any before this insn + } + while next < lines.len() && lines[next].addr == addr { + let _ = writeln!(out, "; line {}", lines[next].line); + next += 1; + } + let _ = writeln!( + out, + "{addr:08x}: {word:08x} {}", + format::word_with(word, addr, &symbols) + ); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips_basic_words() { + // Hand-computed words (no dependency on the encoder module): + // ret = 0xD65F0000 | (30 << 5); add w0,w1,w2; bl .+8; nop. + assert_eq!(one(0xD65F_03C0), "ret"); + assert_eq!(one(0x0B02_0020), "add w0, w1, w2"); + assert_eq!(one(0x8B02_0020), "add x0, x1, x2"); + assert_eq!(one(0x9400_0002), "bl .+8"); + assert_eq!(one(0x1400_0002), "b .+8"); + assert_eq!(one(0xD503_201F), "nop"); + // A word the decoder does not recognise surfaces as a raw word. + assert_eq!( + one(0x0e21_4022 /* addhn: not decoded */), + ".word 0x0e214022" + ); + // ldar/stlr in every width (GNU as output). + assert_eq!(one(0x88df_fc20), "ldar w0, [x1]"); + assert_eq!(one(0xc8df_fc62), "ldar x2, [x3]"); + assert_eq!(one(0x08df_fca4), "ldarb w4, [x5]"); + assert_eq!(one(0x48df_fce6), "ldarh w6, [x7]"); + assert_eq!(one(0x889f_fc20), "stlr w0, [x1]"); + assert_eq!(one(0xc89f_fc62), "stlr x2, [x3]"); + assert_eq!(one(0x089f_fca4), "stlrb w4, [x5]"); + assert_eq!(one(0x489f_fce6), "stlrh w6, [x7]"); + } + + #[test] + fn decodes_objdump_vectors() { + // Spot checks from the Zig suite (verified against objdump there). + assert_eq!(one(0xEB09_0108), "subs x8, x8, x9"); + assert_eq!(one(0x3400_00E8), "cbz w8, .+28"); + assert_eq!(one(0x5400_00C3), "b.cc .+24"); + assert_eq!(one(0xF85F_83A8), "ldur x8, [x29, #-8]"); + assert_eq!(one(0xA941_7BFD), "ldp x29, x30, [sp, #16]"); + assert_eq!(one(0x5300_3D08), "uxth w8, w8"); + assert_eq!(one(0x531F_7908), "lsl w8, w8, #1"); + assert_eq!(one(0x9340_7C00), "sxtw x0, w0"); + assert_eq!(one(0x9280_0AA1), "mov x1, #0xffffffffffffffaa"); + } + + #[test] + fn format_with_lines_interleaves_markers() { + let code = [0x0B02_0020, 0x1B01_7C00, 0xD65F_03C0]; + let lines = [ + SourceLine { offset: 0, line: 2 }, + SourceLine { offset: 4, line: 3 }, + ]; + assert_eq!( + format_with_lines(&code, &lines), + "; line 2\n\ + 0000: 0b020020 add w0, w1, w2\n\ + ; line 3\n\ + 0004: 1b017c00 mul w0, w0, w1\n\ + 0008: d65f03c0 ret\n" + ); + } + + #[test] + fn format_module_annotates_branch_targets() { + // fn f at word 0, fn g at word 3; b .+8 lands at words 2 and 4. + let code = [ + 0x1400_0002, + 0xD65F_03C0, + 0x1400_0002, + 0xD65F_03C0, + 0xD65F_03C0, + ]; + let syms = [Sym { name: "f", word: 0 }, Sym { name: "g", word: 3 }]; + let text = format_module(&code, &syms); + assert!(text.contains("b .+8 "), "{text}"); + assert!(text.contains("b .+8 "), "{text}"); + } + + #[test] + fn format_elf_resolves_absolute_targets() { + let base: u64 = 0x1000; + let code = [0x9400_0002, 0xB000_0000, 0xD65F_03C0]; + let syms = [ + AddrSym { + name: "start", + addr: 0x1000, + size: 8, + is_func: true, + }, + AddrSym { + name: "callee", + addr: 0x1008, + size: 4, + is_func: true, + }, + AddrSym { + name: "gv", + addr: 0x2000, + size: 16, + is_func: false, + }, + ]; + let text = format_elf(&code, base, &syms); + assert!(text.contains("0000000000001000 :"), "{text}"); + assert!(text.contains("0000000000001008 :"), "{text}"); + assert!(text.contains("bl 0x1008 "), "{text}"); + assert!(text.contains("adrp x0, 0x2000 "), "{text}"); + } +} diff --git a/src/target/src/aarch64/lower.rs b/src/target/src/aarch64/lower.rs index 7a58d2f..9bd730d 100644 --- a/src/target/src/aarch64/lower.rs +++ b/src/target/src/aarch64/lower.rs @@ -9,9 +9,10 @@ use crate::regalloc::wimmer::{ }; use alloc::{vec, vec::Vec}; use e::{Cond, FKind, Reg}; -use volt_ir::attribute::{AttrValue, Attribute}; +use volt_ir::attribute::{AttrValue, Attribute, Endianness}; use volt_ir::function::{ - AttrTarget, BinOp, Block, CmpOp, Function, Inst, Jump, Opcode, Ret, Terminator, UnaryOp, Value, + AtomicOrdering, AttrTarget, BinOp, Block, CmpOp, ConvertKind, Function, Inst, Jump, MemFlags, + Opcode, Ret, Terminator, UnaryOp, Value, }; use volt_ir::types::{FloatKind, Type, TypeKind}; @@ -39,9 +40,8 @@ fn quad(f: &Function, v: Value) -> bool { fn vector(f: &Function, v: Value) -> bool { matches!(kind(f, v), TypeKind::Vector(_)) } -fn signed(f: &Function, v: Value) -> bool { - matches!(kind(f,v), TypeKind::Int(i) if i.signed) -} +/// Integer values of fewer than 64 bits are held zero-extended in registers; signedness is a +/// property of the operation, which sign-extends its operands on demand (`signed_operand`). fn bits(f: &Function, v: Value) -> u16 { match kind(f, v) { TypeKind::Int(i) => i.bits, @@ -54,12 +54,24 @@ fn canonical_constant(f: &Function, v: Value, n: u64) -> u64 { let bits = u32::from(bits(f, v)); if bits >= 64 { n - } else if signed(f, v) { - (((n << (64 - bits)) as i64) >> (64 - bits)) as u64 } else { n & ((1u64 << bits) - 1) } } +/// Width in bytes of an `ldar`/`stlr` access to a value of `v`'s type: whole-byte integers up to +/// 64 bits that are a power of two wide, and pointers. Other types have no ordered form. +fn ordered_size(f: &Function, v: Value) -> Option { + match kind(f, v) { + TypeKind::Int(i) if matches!(i.bits, 8 | 16 | 32 | 64) => Some((i.bits / 8) as u8), + TypeKind::Ptr(_) => Some(8), + _ => None, + } +} +/// Whether the access is lowered to `ldar`/`stlr`, which only take a bare `[xn]` address, so no +/// offset may be folded into it. A relaxed access is an ordinary `ldr`/`str` and folds freely. +fn needs_bare_address(mem: &MemFlags) -> bool { + matches!(mem.ordering, Some(o) if o != AtomicOrdering::Relaxed) +} fn wide(f: &Function, v: Value) -> bool { bits(f, v) > 32 } @@ -530,6 +542,15 @@ impl Emitter<'_> { self.frame_imm(false, Reg::X8, base, off)?; Ok((Reg::X8, 0)) } + /// The base register for an `ldar`/`stlr` of `[base + off]`: these have no offset form, so + /// a non-zero `off` is added into the x8 scratch first. + fn bare_base(&mut self, base: Reg, off: usize) -> Result { + if off == 0 { + return Ok(base); + } + self.frame_imm(false, Reg::X8, base, off)?; + Ok(Reg::X8) + } fn loc(&self, v: Value) -> Result { if let Some(&r) = self.alloc.single_regs.get(v.index()) && r != u16::MAX @@ -597,14 +618,14 @@ impl Emitter<'_> { } /// Whether `v = lhs op rhs` (rhs a constant when `imm` is given) is already /// in canonical form, given canonical operands. Values narrower than 64 bits - /// are kept zero- (unsigned) or sign-extended (signed) to the full register. - /// Operations on 32-bit or narrower types run in 32-bit form, which zeroes - /// the upper half, so an unsigned 32-bit result is always canonical; a - /// narrower unsigned result only is if the operation cannot set bits above + /// are kept zero-extended to the full register. Operations on 32-bit or + /// narrower types run in 32-bit form, which zeroes the upper half, so a + /// 32-bit result is always canonical (`SMulh` excepted: it works in 64-bit + /// form); a narrower result only is if the operation cannot set bits above /// the type. fn keeps_canonical_form(&self, v: Value, op: BinOp, imm: Option) -> bool { let n = bits(self.f, v); - if class(self.f, v) != 0 || n >= 64 || signed(self.f, v) || wide(self.f, v) { + if class(self.f, v) != 0 || n >= 64 || wide(self.f, v) || op == BinOp::SMulh { return false; } if n == 32 { @@ -613,7 +634,13 @@ impl Emitter<'_> { let in_range = imm.is_none_or(|k| k >= 0 && (k as u64) >> n == 0); matches!( op, - BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor | BinOp::Shr | BinOp::Div | BinOp::Rem + BinOp::BitAnd + | BinOp::BitOr + | BinOp::BitXor + | BinOp::Shr + | BinOp::UDiv + | BinOp::URem + | BinOp::UMulh ) && in_range } /// `rd = rn op imm` as a single instruction, if the operation has an @@ -622,11 +649,17 @@ impl Emitter<'_> { let is64 = wide(self.f, v); let width = if is64 { 64 } else { 32 }; match op { - BinOp::Shl | BinOp::Shr if (0..width).contains(&imm) => Some(match op { - BinOp::Shl => e::lsl_imm(rd, rn, imm as u32, is64), - _ if signed(self.f, v) => e::asr_imm(rd, rn, imm as u32, is64), - _ => e::lsr_imm(rd, rn, imm as u32, is64), - }), + BinOp::Shl | BinOp::Shr | BinOp::Sar if (0..width).contains(&imm) => { + let n = bits(self.f, v); + Some(match op { + BinOp::Shl => e::lsl_imm(rd, rn, imm as u32, is64), + BinOp::Shr => e::lsr_imm(rd, rn, imm as u32, is64), + _ if n == 32 || n == 64 => e::asr_imm(rd, rn, imm as u32, is64), + // A zero-extended narrow value: extract its bitfield from `imm` up, + // sign-extended from bit `n - 1`. + _ => e::sbfm(rd, rn, imm.min(i64::from(n) - 1) as u8, (n - 1) as u8), + }) + } BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => { let value = if is64 { imm as u64 @@ -646,13 +679,45 @@ impl Emitter<'_> { fn normalize(&mut self, v: Value, r: Reg) { let n = bits(self.f, v); if n < 64 { - self.push(if signed(self.f, v) { - e::sbfm(r, r, 0, (n - 1) as u8) - } else { - e::ubfm(r, r, 0, (n - 1) as u8) - }); + self.push(e::ubfm(r, r, 0, (n - 1) as u8)); } } + /// Whether `signed_operand` has to emit an extension for a value of `v`'s + /// type. Narrow operations read only the low 32 bits of a register, so a + /// 32-bit value needs none unless the whole register is read (`full`). + fn needs_sign_extension(&self, v: Value, full: bool) -> bool { + let n = bits(self.f, v); + n < 64 && (n != 32 || full) + } + /// `r`, which holds `v` zero-extended, as a sign-extended value for an + /// operation that reads its sign: `r` itself when no extension is needed, + /// else `scratch` after an extension into it. + fn signed_operand(&mut self, v: Value, r: Reg, scratch: Reg, full: bool) -> Reg { + if !self.needs_sign_extension(v, full) { + return r; + } + self.push(e::sbfm(scratch, r, 0, (bits(self.f, v) - 1) as u8)); + scratch + } + /// Sets the flags for an integer compare of `a` and `b` (holding `lhs`'s + /// type) and returns the condition that `op` tests. + fn compare_flags(&mut self, op: CmpOp, lhs: Value, a: Reg, b: Reg) -> Result { + let cond = int_cond(op)?; + let (a, b) = if matches!(op, CmpOp::Slt | CmpOp::Sle | CmpOp::Sgt | CmpOp::Sge) { + ( + self.signed_operand(lhs, a, reg(16), false), + self.signed_operand(lhs, b, reg(17), false), + ) + } else { + (a, b) + }; + self.push(if wide(self.f, lhs) { + e::cmp64(a, b) + } else { + e::cmp(a, b) + }); + Ok(cond) + } fn move_one(&mut self, m: Move) -> Result<(), Error> { if m.src == m.dst { return Ok(()); @@ -759,18 +824,15 @@ impl Emitter<'_> { e::str_fp(r, base, off, double(self.f, v)) } } else { - match (load, size, signed(self.f, v)) { - (true, 1, true) => e::ldrsb_off(r, base, off), - (true, 1, false) => e::ldrb_off(r, base, off), - (true, 2, true) => e::ldrsh_off(r, base, off), - (true, 2, false) => e::ldrh_off(r, base, off), - (true, 3..=4, true) => e::ldrsw(r, base, off), - (true, 3..=4, false) => e::ldr_w(r, base, off), - (true, 5..=8, _) => e::ldr_off(r, base, off), - (false, 1, _) => e::strb_off(r, base, off), - (false, 2, _) => e::strh_off(r, base, off), - (false, 3..=4, _) => e::str_w(r, base, off), - (false, 5..=8, _) => e::str_off(r, base, off), + match (load, size) { + (true, 1) => e::ldrb_off(r, base, off), + (true, 2) => e::ldrh_off(r, base, off), + (true, 3..=4) => e::ldr_w(r, base, off), + (true, 5..=8) => e::ldr_off(r, base, off), + (false, 1) => e::strb_off(r, base, off), + (false, 2) => e::strh_off(r, base, off), + (false, 3..=4) => e::str_w(r, base, off), + (false, 5..=8) => e::str_off(r, base, off), _ => return Err(Error::Unsupported), } }; @@ -789,50 +851,67 @@ impl Emitter<'_> { b: Reg, ) -> Result<(), Error> { let sf = if wide(self.f, v) { 1 << 31 } else { 0 }; - let si = signed(self.f, v); let w = match op { BinOp::Add => e::add(rd, a, b), BinOp::Sub => e::sub(rd, a, b), BinOp::Mul => e::mul(rd, a, b), - BinOp::Mulh => { - if bits(self.f, v) != 64 { + BinOp::UMulh | BinOp::SMulh => { + let n = bits(self.f, v); + if n == 64 { + self.push(if op == BinOp::SMulh { + e::smulh(rd, a, b) + } else { + e::umulh(rd, a, b) + }); + return Ok(()); + } + // The product of two values of up to 32 bits fits a register. + if n > 32 { return Err(Error::Unsupported); } - if si { - e::smulh(rd, a, b) + if op == BinOp::SMulh { + let a = self.signed_operand(v, a, reg(16), true); + let b = self.signed_operand(v, b, reg(17), true); + self.push(e::mul64(rd, a, b)); + self.push(e::asr_imm(rd, rd, u32::from(n), true)); } else { - e::umulh(rd, a, b) + self.push(e::mul64(rd, a, b)); + self.push(e::lsr_imm(rd, rd, u32::from(n), true)); } + return Ok(()); } - BinOp::Div => { - if si { - e::sdiv(rd, a, b) - } else { - e::udiv(rd, a, b) - } + BinOp::UDiv => e::udiv(rd, a, b), + BinOp::SDiv => { + let a = self.signed_operand(v, a, reg(16), false); + let b = self.signed_operand(v, b, reg(17), false); + e::sdiv(rd, a, b) } - BinOp::Rem => { - self.push( - sf | if si { - e::sdiv(reg(16), a, b) - } else { - e::udiv(reg(16), a, b) - }, - ); + BinOp::URem => { + self.push(sf | e::udiv(reg(16), a, b)); self.push(sf | e::msub(rd, reg(16), b, a)); return Ok(()); } + BinOp::SRem => { + // Extended operands live in x16/x17, so the quotient can use `rd`. + let extend = self.needs_sign_extension(v, false); + let a = self.signed_operand(v, a, reg(16), false); + let b = self.signed_operand(v, b, reg(17), false); + let q = if extend { rd } else { reg(16) }; + self.push(sf | e::sdiv(q, a, b)); + self.push(sf | e::msub(rd, q, b, a)); + return Ok(()); + } BinOp::BitAnd => e::andr(rd, a, b), BinOp::BitOr => e::orr(rd, a, b), BinOp::BitXor => e::eor(rd, a, b), BinOp::Shl => e::lslv(rd, a, b), - BinOp::Shr => { - if si { - e::asrv(rd, a, b) - } else { - e::lsrv(rd, a, b) - } + BinOp::Shr => e::lsrv(rd, a, b), + BinOp::Sar => { + let a = self.signed_operand(v, a, reg(16), false); + e::asrv(rd, a, b) } + // Floating-point only. + BinOp::Div | BinOp::Rem => return Err(Error::Unsupported), }; self.push(sf | w); Ok(()) @@ -915,7 +994,7 @@ impl Emitter<'_> { } self.finish(v, rd) } - fn convert(&mut self, v: Value, src: Value) -> Result<(), Error> { + fn convert(&mut self, v: Value, src: Value, kind: ConvertKind) -> Result<(), Error> { if quad(self.f, v) || quad(self.f, src) || vector(self.f, v) || vector(self.f, src) { return Err(Error::Unsupported); } @@ -923,25 +1002,43 @@ impl Emitter<'_> { let dc = class(self.f, v); let a = self.read(src, reg(if sc == 1 { 24 } else { 13 }))?; let rd = self.result(v)?; - match (sc, dc) { - (0, 1) => { - self.push(e::cvt_int_to_float( - rd, - a, - fkind(self.f, v, self.caps), - signed(self.f, src), - )); + match kind { + ConvertKind::SiToFp | ConvertKind::UiToFp => { + if (sc, dc) != (0, 1) { + return Err(Error::Unsupported); + } + let si = kind == ConvertKind::SiToFp; + let a = if si { + self.signed_operand(src, a, reg(16), false) + } else { + a + }; + let sf = if wide(self.f, src) { 1 << 31 } else { 0 }; + self.push(sf | e::cvt_int_to_float(rd, a, fkind(self.f, v, self.caps), si)); if half(self.f, v) && !self.caps.fp16 { self.half_round(rd); } } - (1, 0) => self.push(e::cvt_float_to_int( - rd, - a, - fkind(self.f, src, self.caps), - signed(self.f, v), - )), - (1, 1) => { + ConvertKind::FpToSi | ConvertKind::FpToUi => { + if (sc, dc) != (1, 0) { + return Err(Error::Unsupported); + } + // A destination narrower than 32 bits keeps the low bits of the 32-bit + // saturating conversion (`finish` truncates). + let sf = if wide(self.f, v) { 1 << 31 } else { 0 }; + self.push( + sf | e::cvt_float_to_int( + rd, + a, + fkind(self.f, src, self.caps), + kind == ConvertKind::FpToSi, + ), + ); + } + ConvertKind::FpResize => { + if (sc, dc) != (1, 1) { + return Err(Error::Unsupported); + } if self.caps.fp16 && (half(self.f, src) || half(self.f, v)) { self.push(if half(self.f, src) && half(self.f, v) { e::fmov_reg(rd, a) @@ -969,16 +1066,27 @@ impl Emitter<'_> { self.push(e::fcvt(rd, a, double(self.f, v))); } } - _ => { + ConvertKind::Trunc | ConvertKind::Zext | ConvertKind::Sext => { + if (sc, dc) != (0, 0) { + return Err(Error::Unsupported); + } let n = bits(self.f, src); - if bits(self.f, v) > n && n < 64 { - self.push(if signed(self.f, src) { - e::sbfm(rd, a, 0, (n - 1) as u8) - } else { - e::ubfm(rd, a, 0, (n - 1) as u8) - }); + let m = bits(self.f, v); + if kind == ConvertKind::Sext && m > n && n < 64 { + let word = e::sbfm(rd, a, 0, (n - 1) as u8); + if m == 32 { + // The 32-bit form clears the upper half: already canonical. + self.push(word & !(1 << 31 | 1 << 22)); + return self.finish_normal(v, rd); + } + self.push(word); } else { + // A zero-extension of a canonical value, a truncation (`finish` + // clears the bits above `m`) or a same-width copy. self.push(e::mov(rd, a)); + if kind == ConvertKind::Zext { + return self.finish_normal(v, rd); + } } } } @@ -1132,19 +1240,12 @@ impl Emitter<'_> { if let Fusion::Compare { lhs, rhs, op } = fusion { let a = self.read(lhs, reg(13))?; let b = self.read(rhs, reg(14))?; - self.push(if wide(self.f, lhs) { - e::cmp64(a, b) - } else { - e::cmp(a, b) - }); + let cond = self.compare_flags(op, lhs, a, b)?; let Opcode::If(cf) = self.f.opcode_ref(inst) else { return Err(Error::Unsupported); }; let other = self.local_label(); - self.branch( - other, - e::bcc(e::invert(int_cond(op, signed(self.f, lhs))), 0), - ); + self.branch(other, e::bcc(e::invert(cond), 0)); self.edge(cf.then)?; self.bind(other); self.edge(cf.else_)?; @@ -1222,7 +1323,7 @@ impl Emitter<'_> { } else if let Some(word) = self.immediate_form(v, a.op, rd, r, a.imm) { self.push(word); } else { - self.constant(reg(14), a.imm as u64); + self.constant(reg(14), canonical_constant(self.f, v, a.imm as u64)); self.integer_binary(v, a.op, rd, r, reg(14))?; } if self.keeps_canonical_form(v, a.op, Some(a.imm)) { @@ -1247,6 +1348,7 @@ impl Emitter<'_> { CmpOp::Le => e::fcmge_vec(rd, b, a), CmpOp::Gt => e::fcmgt_vec(rd, a, b), CmpOp::Ge => e::fcmge_vec(rd, a, b), + _ => return Err(Error::Unsupported), }); if c.op == CmpOp::Ne { self.push(e::mvn_vec(rd, rd)); @@ -1255,15 +1357,12 @@ impl Emitter<'_> { if quad(self.f, c.lhs) { return Err(Error::Unsupported); } + let cond = float_cond(c.op)?; self.push(e::fcmp(a, b, fkind(self.f, c.lhs, self.caps))); - self.push(e::cset(rd, float_cond(c.op))); + self.push(e::cset(rd, cond)); } else { - self.push(if wide(self.f, c.lhs) { - e::cmp64(a, b) - } else { - e::cmp(a, b) - }); - self.push(e::cset(rd, int_cond(c.op, signed(self.f, c.lhs)))); + let cond = self.compare_flags(c.op, c.lhs, a, b)?; + self.push(e::cset(rd, cond)); } // `cset` already yields 0 or 1. if !fp && !vector(self.f, c.lhs) && bits(self.f, v) == 1 { @@ -1297,15 +1396,17 @@ impl Emitter<'_> { e::csel(rd, a, b, Cond::Ne) | if wide(self.f, v) { 1 << 31 } else { 0 } }); } - // `csel` copies one of two canonical operands, so an unsigned - // result needs no extension. - if class(self.f, v) == 0 && !signed(self.f, v) && !vector(self.f, v) { + // `csel` copies one of two canonical operands, so the result + // needs no extension. + if class(self.f, v) == 0 && !vector(self.f, v) { self.finish_normal(v, rd)?; } else { self.finish(v, rd)?; } } - Opcode::Convert(c) => self.convert(result.ok_or(Error::Unsupported)?, c.value)?, + Opcode::Convert(c) => { + self.convert(result.ok_or(Error::Unsupported)?, c.value, c.kind)? + } Opcode::Unary(u) => { let v = result.ok_or(Error::Unsupported)?; let rd = self.result(v)?; @@ -1370,25 +1471,73 @@ impl Emitter<'_> { self.finish(v, rd)?; } Opcode::Load(l) => { + if l.mem.endian == Endianness::Big { + return Err(Error::Unsupported); + } let v = result.ok_or(Error::Unsupported)?; + let size = ordered_size(self.f, v); + let acquire = match l.mem.ordering { + None => false, + // Relaxed: an ordinary, naturally aligned load is single-copy atomic. + Some(AtomicOrdering::Relaxed) => { + if size.is_none() { + return Err(Error::Unsupported); + } + false + } + Some(AtomicOrdering::Acquire | AtomicOrdering::SeqCst) => true, + Some(AtomicOrdering::Release | AtomicOrdering::AcqRel) => { + return Err(Error::Unsupported); + } + }; let a = self.read(l.ptr, reg(13))?; let rd = self.result(v)?; - self.memory(true, v, rd, a, self.fold.off_of(inst) as usize)?; - // `ldrb`/`ldrh`/`ldr w` already zero-extend a whole-byte unsigned value. + if acquire { + let size = size.ok_or(Error::Unsupported)?; + let base = self.bare_base(a, self.fold.off_of(inst) as usize)?; + self.push(e::ldar(rd, base, size)); + } else { + self.memory(true, v, rd, a, self.fold.off_of(inst) as usize)?; + } + // `ldrb`/`ldrh`/`ldr w`/`ldar*` already zero-extend a whole-byte value. let n = bits(self.f, v); - if class(self.f, v) == 0 && !signed(self.f, v) && matches!(n, 8 | 16 | 32 | 64) { + if class(self.f, v) == 0 && matches!(n, 8 | 16 | 32 | 64) { self.finish_normal(v, rd)?; } else { self.finish(v, rd)?; } } Opcode::Store(s) => { + if s.mem.endian == Endianness::Big { + return Err(Error::Unsupported); + } + let size = ordered_size(self.f, s.value); + let release = match s.mem.ordering { + None => false, + // Relaxed: an ordinary, naturally aligned store is single-copy atomic. + Some(AtomicOrdering::Relaxed) => { + if size.is_none() { + return Err(Error::Unsupported); + } + false + } + Some(AtomicOrdering::Release | AtomicOrdering::SeqCst) => true, + Some(AtomicOrdering::Acquire | AtomicOrdering::AcqRel) => { + return Err(Error::Unsupported); + } + }; let a = self.read( s.value, reg(if class(self.f, s.value) == 1 { 24 } else { 13 }), )?; let p = self.read(s.ptr, reg(14))?; - self.memory(false, s.value, a, p, self.fold.off_of(inst) as usize)?; + if release { + let size = size.ok_or(Error::Unsupported)?; + let base = self.bare_base(p, self.fold.off_of(inst) as usize)?; + self.push(e::stlr(a, base, size)); + } else { + self.memory(false, s.value, a, p, self.fold.off_of(inst) as usize)?; + } } Opcode::Prefetch(p) => { let a = self.read(p.ptr, reg(13))?; @@ -1464,7 +1613,12 @@ impl Emitter<'_> { } else { e::umov_lane(rd, a, x.index as u8) }); - self.finish(v, rd)?; + if fp { + self.finish(v, rd)?; + } else { + // `umov` to a w register zero-extends the 32-bit lane. + self.finish_normal(v, rd)?; + } } Opcode::StructNew(s) => { let v = result.ok_or(Error::Unsupported)?; @@ -1520,12 +1674,10 @@ impl Emitter<'_> { let b = self.read(d.a, reg(25))?; let c = self.read(d.b, reg(26))?; self.push(e::mov_vec(reg(27), a)); - let si = match kind(self.f, d.a) { - TypeKind::Vector(x) => { - matches!(self.f.types.type_kind(x.elem),TypeKind::Int(i) if i.signed) - } - _ => return Err(Error::Unsupported), - }; + if !vector(self.f, d.a) { + return Err(Error::Unsupported); + } + let si = d.signed; self.push(if si { e::sdot(reg(27), b, c) } else { @@ -1601,49 +1753,33 @@ impl Emitter<'_> { Ok(()) } } -fn int_cond(op: CmpOp, si: bool) -> Cond { - match op { +/// The condition code that holds after comparing the operands of an integer +/// (or pointer) `op`; the signed forms need sign-extended operands. +fn int_cond(op: CmpOp) -> Result { + Ok(match op { CmpOp::Eq => Cond::Eq, CmpOp::Ne => Cond::Ne, - CmpOp::Lt => { - if si { - Cond::Lt - } else { - Cond::Lo - } - } - CmpOp::Le => { - if si { - Cond::Le - } else { - Cond::Ls - } - } - CmpOp::Gt => { - if si { - Cond::Gt - } else { - Cond::Hi - } - } - CmpOp::Ge => { - if si { - Cond::Ge - } else { - Cond::Hs - } - } - } + CmpOp::Slt => Cond::Lt, + CmpOp::Sle => Cond::Le, + CmpOp::Sgt => Cond::Gt, + CmpOp::Sge => Cond::Ge, + CmpOp::Ult => Cond::Lo, + CmpOp::Ule => Cond::Ls, + CmpOp::Ugt => Cond::Hi, + CmpOp::Uge => Cond::Hs, + CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge => return Err(Error::Unsupported), + }) } -fn float_cond(op: CmpOp) -> Cond { - match op { +fn float_cond(op: CmpOp) -> Result { + Ok(match op { CmpOp::Eq => Cond::Eq, CmpOp::Ne => Cond::Ne, CmpOp::Lt => Cond::Mi, CmpOp::Le => Cond::Ls, CmpOp::Gt => Cond::Gt, CmpOp::Ge => Cond::Ge, - } + _ => return Err(Error::Unsupported), + }) } /// True for a function made only of scalar integer, boolean and pointer @@ -1671,6 +1807,14 @@ fn is_plain_integer(f: &Function) -> bool { ) { return false; } + let mem = match f.opcode_ref(*i) { + Opcode::Load(l) => Some(&l.mem), + Opcode::Store(s) => Some(&s.mem), + _ => None, + }; + if mem.is_some_and(|m| !m.is_plain()) { + return false; + } } } (0..f.value_count()).all(|v| match kind(f, Value(v as u32)) { @@ -1709,13 +1853,16 @@ pub(super) fn compile_owned( _ => false, }); volt_ir::softfp::lower(&mut f); + // Legalize rewrites byte-swapped and atomic float accesses into forms selected below; + // ordered int/pointer accesses become `ldar`/`stlr` (or plain accesses when relaxed). + volt_ir::legalize::legalize(&mut f); volt_ir::critical_edge::split_critical_edges(&mut f); volt_ir::reachable::neutralize_unreachable(&mut f); } let fold = crate::regalloc::addrfold::analyze(&f, |f, i| { let (p, v) = match f.opcode_ref(i) { - Opcode::Load(l) => (l.ptr, f.inst_result(i)?), - Opcode::Store(s) => (s.ptr, s.value), + Opcode::Load(l) if !needs_bare_address(&l.mem) => (l.ptr, f.inst_result(i)?), + Opcode::Store(s) if !needs_bare_address(&s.mem) => (s.ptr, s.value), _ => return None, }; let add = match f.opcode_ref(f.defining_inst(p)?) { diff --git a/src/target/src/aarch64/mod.rs b/src/target/src/aarch64/mod.rs index 2ed3b76..3fcfa94 100644 --- a/src/target/src/aarch64/mod.rs +++ b/src/target/src/aarch64/mod.rs @@ -1,5 +1,4 @@ //! AArch64 encoding, Function-based native selection, object linking and JIT. -pub mod disasm; pub mod encode; pub mod isel; #[cfg(all( @@ -11,5 +10,6 @@ pub mod isel; ))] pub mod jit; pub mod link; +pub mod listing; pub mod object; pub mod peephole; diff --git a/src/target/src/aarch64/object.rs b/src/target/src/aarch64/object.rs index cd1da2c..3421e31 100644 --- a/src/target/src/aarch64/object.rs +++ b/src/target/src/aarch64/object.rs @@ -274,8 +274,8 @@ fn reloc_flags(reloc_type: RelocType) -> object::RelocationFlags { /// are emitted as undefined extern symbols. pub fn write_module_object(linked: &super::link::Linked) -> Vec { use super::isel::RelocKind; - use super::link::DataKind; use alloc::collections::BTreeMap; + use volt_ir::module::DataKind; let mut symbols: Vec = Vec::new(); let mut id_of_name: BTreeMap<&str, u32> = BTreeMap::new(); @@ -298,15 +298,17 @@ pub fn write_module_object(linked: &super::link::Linked) -> Vec { for d in linked.data.iter() { let section = match d.kind { DataKind::Rodata => { + rodata_bytes.resize(d.off, 0); rodata_bytes.extend_from_slice(&d.bytes); SectionKind::Rodata } DataKind::Data => { + data_bytes.resize(d.off, 0); data_bytes.extend_from_slice(&d.bytes); SectionKind::Data } DataKind::Bss => { - bss_size += d.size as u64; + bss_size = (d.off + d.size) as u64; SectionKind::Bss } }; @@ -370,7 +372,7 @@ pub fn write_module_object(linked: &super::link::Linked) -> Vec { for r in d.relocs.iter() { data_relocs.push(DataRelocEntry { section, - offset: (d.off + r.off) as u64, + offset: (d.off + r.offset) as u64, symbol: id_of_name[r.symbol.as_str()], addend: 0, }); @@ -391,9 +393,12 @@ pub fn write_module_object(linked: &super::link::Linked) -> Vec { #[cfg(test)] mod tests { use super::*; - use crate::aarch64::link::{DataReloc, Linked}; + use crate::aarch64::isel::ModelCaps; + use crate::aarch64::link::{Linked, compile_module}; use alloc::vec; - use object::read::{Object as _, ObjectSymbol as _}; + use object::read::{Object as _, ObjectSection as _, ObjectSymbol as _}; + use volt_ir::function::{Function, Opcode, Ret, Terminator}; + use volt_ir::module::{DataReloc, Module}; #[test] fn elf_round_trip_aarch64() { @@ -474,9 +479,47 @@ mod tests { assert_eq!(obj.architecture(), object::Architecture::Aarch64); assert!(obj.sections().count() >= 2); assert!(obj.symbols().count() >= 2, "defined + extern symbols"); - let _ = DataReloc { - off: 0, - symbol: String::new(), - }; + } + + #[test] + fn elf_from_an_ir_module_with_globals() { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let v = f.append_inst(block, ty, Opcode::Iconst(5)); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + let mut m = Module::new(); + m.add_function("five", f); + m.add_data("ro", vec![1, 2, 3, 4]); + m.add_data("ro8", vec![0; 8]); + m.add_writable_relocs( + "table", + vec![0; 16], + vec![DataReloc { + offset: 8, + symbol: "five".to_string(), + }], + ); + m.add_bss("scratch", 32); + let linked = compile_module(&m, &ModelCaps::default()).unwrap(); + let bytes = write_module_object(&linked); + let obj = object::read::File::parse(&bytes[..]).expect("module ELF parses"); + assert_eq!(obj.architecture(), object::Architecture::Aarch64); + let defined: Vec = obj + .symbols() + .filter(|s| s.is_definition()) + .filter_map(|s| s.name().ok().map(|n| n.to_string())) + .collect(); + for want in ["five", "ro", "table", "scratch"] { + assert!(defined.iter().any(|n| n == want), "symbol {want} defined"); + } + let data = obj.section_by_name(".data").expect(".data section"); + assert_eq!(data.data().unwrap().len(), 16); + // `ro8` is placed at offset 8, so `ro` is padded to it. + let rodata = obj.section_by_name(".data.rel.ro").expect("rodata section"); + assert_eq!(rodata.data().unwrap().len(), 16); + let relocs: Vec<_> = data.relocations().collect(); + assert_eq!(relocs.len(), 1); + assert_eq!(relocs[0].0, 8, "ABS64 slot offset within .data"); } } diff --git a/src/target/src/lib.rs b/src/target/src/lib.rs index ca51096..3f25aeb 100644 --- a/src/target/src/lib.rs +++ b/src/target/src/lib.rs @@ -12,24 +12,3 @@ pub mod x86_64; pub mod native; pub mod regalloc; - -use core::fmt::Debug; - -/// Failure to decode a guest instruction. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct DecodeError { - /// Original instruction bits, useful for diagnostics and guest exceptions. - pub word: W, -} - -/// Architecture-specific decoder. `W` is the instruction word representation; -/// variable-length ISAs can use a byte slice in their own decoder implementation. -pub trait Decoder { - /// Encoded instruction word. - type Word: Copy + Debug; - /// Decoded architecture-specific instruction. - type Instruction: Debug; - - /// Decode one instruction or reject the encoding. - fn decode(&self, word: Self::Word) -> Result>; -} diff --git a/src/target/src/native.rs b/src/target/src/native.rs index dae8301..7ac7eaa 100644 --- a/src/target/src/native.rs +++ b/src/target/src/native.rs @@ -23,10 +23,7 @@ pub use module::map_linked_aarch64; all(target_arch = "x86_64", feature = "x86_64"), all(target_arch = "aarch64", feature = "aarch64") ))] -pub use module::{ - DataKind, DataReloc, Error as ModuleError, JittedModule, ModuleData, ModuleFunction, - jit_module, jit_module_data, -}; +pub use module::{Error as ModuleError, JittedModule, jit_module}; #[derive(Debug)] pub enum Error { diff --git a/src/target/src/native/module.rs b/src/target/src/native/module.rs index 37a3072..df2b60d 100644 --- a/src/target/src/native/module.rs +++ b/src/target/src/native/module.rs @@ -3,7 +3,7 @@ use super::{Error as NativeError, JittedFunction, sync_icache}; use core::{fmt, ptr::NonNull}; -use volt_ir::function::Function; +use volt_ir::module::{DataKind, Global, Module}; #[cfg(target_arch = "aarch64")] use crate::aarch64::isel; @@ -48,40 +48,14 @@ impl From for Error { } } -pub struct ModuleFunction<'a> { - pub name: &'a str, - pub function: &'a Function, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum DataKind { - Rodata, - Data, - Bss, -} -impl DataKind { - fn section(self) -> usize { - match self { - Self::Rodata => 0, - Self::Data => 1, - Self::Bss => 2, - } +fn section_index(kind: DataKind) -> usize { + match kind { + DataKind::Rodata => 0, + DataKind::Data => 1, + DataKind::Bss => 2, } } -pub struct DataReloc<'a> { - pub offset: usize, - pub symbol: &'a str, -} - -pub struct ModuleData<'a> { - pub name: &'a str, - pub bytes: &'a [u8], - pub kind: DataKind, - pub size: usize, - pub relocs: &'a [DataReloc<'a>], -} - /// One contiguous image. Code is RX, constants R, mutable globals and BSS RW. pub struct JittedModule { image: JittedFunction, @@ -133,32 +107,27 @@ fn symbol_offset( .ok_or_else(|| Error::UndefinedSymbol(name.into())) } -pub fn jit_module(functions: &[ModuleFunction<'_>]) -> Result { - jit_module_data(functions, &[]) -} - -pub fn jit_module_data( - functions: &[ModuleFunction<'_>], - data: &[ModuleData<'_>], -) -> Result { +/// Compile `module` for the host, map it as one image, and resolve every +/// intra-module call, `global_addr`, and data relocation. +pub fn jit_module(module: &Module) -> Result { #[cfg(not(target_os = "linux"))] { - let _ = (functions, data); + let _ = module; return Err(NativeError::UnsupportedHost.into()); } #[cfg(target_os = "linux")] { - let mut compiled = Vec::with_capacity(functions.len()); - let mut symbols = Vec::with_capacity(functions.len()); + let mut compiled = Vec::with_capacity(module.functions.len()); + let mut symbols = Vec::with_capacity(module.functions.len()); let mut code_len = 0usize; - for function in functions { + for def in &module.functions { #[cfg(target_arch = "aarch64")] - let part = isel::compile_function(function.function, &isel::ModelCaps::default()) + let part = isel::compile_function(&def.function, &isel::ModelCaps::default()) .map_err(|e| Error::Compile(Box::new(e)))?; #[cfg(target_arch = "x86_64")] let part = - isel::compile_object(function.function).map_err(|e| Error::Compile(Box::new(e)))?; - symbols.push((function.name.to_owned(), code_len)); + isel::compile_object(&def.function).map_err(|e| Error::Compile(Box::new(e)))?; + symbols.push((def.name.clone(), code_len)); #[cfg(target_arch = "aarch64")] let length = part .code @@ -179,7 +148,7 @@ pub fn jit_module_data( map_module( code_len, symbols, - data, + &module.globals, |memory, base, symbols, data_symbols| { #[cfg(target_arch = "x86_64")] memory[..code_len].fill(0x90); @@ -293,7 +262,7 @@ pub fn jit_module_data( fn map_module( code_len: usize, symbols: Vec<(String, usize)>, - data: &[ModuleData<'_>], + data: &[Global], fill_code: impl FnOnce( &mut [u8], usize, @@ -308,47 +277,42 @@ fn map_module( for name in symbols .iter() .map(|(name, _)| name.as_str()) - .chain(data.iter().map(|d| d.name)) + .chain(data.iter().map(|d| d.name.as_str())) { if !names.insert(name) { return Err(Error::DuplicateSymbol(name.into())); } } + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + if page <= 0 { + return Err(NativeError::Mapping(std::io::Error::last_os_error()).into()); + } + let page = page as usize; let mut extents = [0usize; 3]; let mut placements = Vec::with_capacity(data.len()); for object in data { - if (object.kind != DataKind::Bss && object.size != object.bytes.len()) + if (object.kind != DataKind::Bss && object.size != object.bytes.len() as u64) || (object.kind == DataKind::Bss && !object.bytes.is_empty()) + || (object.align != 0 && !object.align.is_power_of_two()) { return Err(Error::InvalidData); } - let section = object.kind.section(); - let natural = object - .size - .max(1) - .checked_next_power_of_two() - .unwrap_or(8) - .min(8); - let offset = align(extents[section], natural)?; - extents[section] = offset - .checked_add(object.size) - .ok_or(Error::AddressOverflow)?; - for reloc in object.relocs { - if reloc - .offset - .checked_add(8) - .is_none_or(|end| end > object.size) - { + // Each section starts page-aligned, so no stricter alignment is honorable. + let alignment = object.effective_align(); + if alignment > page { + return Err(Error::InvalidData); + } + let size = usize::try_from(object.size).map_err(|_| Error::AddressOverflow)?; + let section = section_index(object.kind); + let offset = align(extents[section], alignment)?; + extents[section] = offset.checked_add(size).ok_or(Error::AddressOverflow)?; + for reloc in &object.relocs { + if reloc.offset.checked_add(8).is_none_or(|end| end > size) { return Err(Error::InvalidRelocation); } } placements.push(offset); } - let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; - if page <= 0 { - return Err(NativeError::Mapping(std::io::Error::last_os_error()).into()); - } - let page = page as usize; let mut starts = [0usize; 3]; let mut mapped_len = align(code_len, page)?; for section in 0..3 { @@ -384,14 +348,14 @@ fn map_module( let memory = unsafe { core::slice::from_raw_parts_mut(address.as_ptr(), mapped_len) }; let mut data_symbols = Vec::with_capacity(data.len()); for (object, placement) in data.iter().zip(&placements) { - let offset = starts[object.kind.section()] + placement; - memory[offset..offset + object.bytes.len()].copy_from_slice(object.bytes); - data_symbols.push((object.name.to_owned(), offset)); + let offset = starts[section_index(object.kind)] + placement; + memory[offset..offset + object.bytes.len()].copy_from_slice(&object.bytes); + data_symbols.push((object.name.clone(), offset)); } let base = address.as_ptr() as usize; for (object, (_, offset)) in data.iter().zip(&data_symbols) { - for reloc in object.relocs { - let target = base + symbol_offset(&symbols, &data_symbols, reloc.symbol)?; + for reloc in &object.relocs { + let target = base + symbol_offset(&symbols, &data_symbols, &reloc.symbol)?; let site = offset + reloc.offset; memory[site..site + 8].copy_from_slice(&(target as u64).to_le_bytes()); } @@ -447,34 +411,16 @@ pub fn map_linked_aarch64(linked: &crate::aarch64::link::Linked) -> Result> = linked + let objects: Vec = linked .data .iter() - .map(|object| { - object - .relocs - .iter() - .map(|r| DataReloc { - offset: r.off, - symbol: r.symbol.as_str(), - }) - .collect() - }) - .collect(); - let objects: Vec<_> = linked - .data - .iter() - .zip(&data_relocs) - .map(|(object, relocs)| ModuleData { - name: &object.name, - bytes: &object.bytes, - kind: match object.kind { - link::DataKind::Rodata => DataKind::Rodata, - link::DataKind::Data => DataKind::Data, - link::DataKind::Bss => DataKind::Bss, - }, - size: object.size, - relocs, + .map(|object| Global { + name: object.name.clone(), + kind: object.kind, + bytes: object.bytes.clone(), + size: object.size as u64, + align: object.align, + relocs: object.relocs.clone(), }) .collect(); map_module( @@ -520,3 +466,197 @@ pub fn map_linked_aarch64(linked: &crate::aarch64::link::Linked) -> Result u64; + type NullaryFn = extern "C" fn() -> u64; + + /// `name(x) = x + 7`. + fn add_seven() -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let x = f.append_block_param(block, ty); + let r = f.append_inst( + block, + ty, + Opcode::ArithImm(ArithImm { + op: BinOp::Add, + lhs: x, + imm: 7, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(r))); + f + } + + /// `name(x) = callee(x) * 3`. + fn triple_of(callee: &str) -> Function { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let x = f.append_block_param(block, ty); + let c = f.append_call(block, ty, callee, &[x]); + let r = f.append_inst( + block, + ty, + Opcode::ArithImm(ArithImm { + op: BinOp::Mul, + lhs: c, + imm: 3, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(r))); + f + } + + /// `name() = *global`, a u64 load through `global_addr`. + fn load_global(global: &str) -> Function { + let mut f = Function::new(); + let u64_t = f.types.parse_type("i64").unwrap(); + let ptr_t = f.types.parse_type("ptr").unwrap(); + let block = f.append_block(); + let symbol = f.intern_symbol(global); + let addr = f.append_inst( + block, + ptr_t, + Opcode::GlobalAddr(GlobalAddr { + symbol, + via_got: false, + }), + ); + let v = f.append_load(block, u64_t, addr, MemFlags::new()); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + f + } + + fn sample_module() -> Module { + let mut m = Module::new(); + m.add_function("caller", triple_of("callee")); + m.add_function("callee", add_seven()); + m.add_function("magic", load_global("constant")); + m.add_data("constant", 0x1122_3344_5566_7788u64.to_le_bytes().to_vec()); + m.add_writable_relocs( + "table", + vec![0; 16], + vec![DataReloc { + offset: 8, + symbol: "callee".to_string(), + }], + ); + m.add_bss("scratch", 24); + m + } + + #[test] + fn functions_call_each_other_and_data_relocates_to_a_function() { + let jit = match jit_module(&sample_module()) { + Ok(jit) => jit, + Err(e) => panic!("jit_module failed: {e}"), + }; + // SAFETY: the signatures match the IR functions above and the host ISA + // is the one this module was compiled for. + unsafe { + let caller: UnaryFn = jit.entry("caller").unwrap(); + let callee: UnaryFn = jit.entry("callee").unwrap(); + assert_eq!(callee(5), 12); + assert_eq!(caller(5), 36); + let magic: NullaryFn = jit.entry("magic").unwrap(); + assert_eq!(magic(), 0x1122_3344_5566_7788); + + // The data relocation holds the absolute address of `callee`. + let table = jit.data_addr("table").unwrap(); + assert_eq!( + core::slice::from_raw_parts(table, 8), + &[0; 8], + "slot without a relocation stays as initialized" + ); + let slot = table.add(8).cast::().read_unaligned(); + assert_eq!(slot, callee as usize as u64); + let through_table: UnaryFn = core::mem::transmute::(slot as usize); + assert_eq!(through_table(1), 8); + + // Writable globals are really writable and BSS starts zeroed. + let scratch = jit.data_addr("scratch").unwrap(); + assert_eq!(core::slice::from_raw_parts(scratch, 24), &[0; 24]); + scratch.write(0xAB); + assert_eq!(scratch.read(), 0xAB); + } + assert!(jit.data_addr("callee").is_none(), "functions are not data"); + assert!(jit.data_addr("nothing").is_none()); + } + + #[test] + fn rejects_undefined_data_relocation_and_duplicate_names() { + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_data_relocs( + "t", + vec![0; 8], + vec![DataReloc { + offset: 0, + symbol: "missing".to_string(), + }], + ); + assert!(matches!( + jit_module(&m), + Err(Error::UndefinedSymbol(name)) if name == "missing" + )); + + let mut m = Module::new(); + m.add_function("same", add_seven()); + m.add_data("same", vec![0; 8]); + assert!(matches!( + jit_module(&m), + Err(Error::DuplicateSymbol(name)) if name == "same" + )); + } + + #[test] + fn rejects_malformed_globals() { + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_data("d", vec![0; 4]); + m.globals[0].size = 8; + assert!(matches!(jit_module(&m), Err(Error::InvalidData))); + + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_data_relocs( + "short", + vec![0; 4], + vec![DataReloc { + offset: 0, + symbol: "f".to_string(), + }], + ); + assert!(matches!(jit_module(&m), Err(Error::InvalidRelocation))); + + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_bss("odd", 8); + m.globals[0].align = 3; + assert!(matches!(jit_module(&m), Err(Error::InvalidData))); + } + + #[test] + fn explicit_alignment_is_honored() { + let mut m = Module::new(); + m.add_function("f", add_seven()); + m.add_writable("pad", vec![1]); + m.add_writable("aligned", vec![0; 8]); + m.globals[1].align = 64; + let jit = match jit_module(&m) { + Ok(jit) => jit, + Err(e) => panic!("jit_module failed: {e}"), + }; + assert_eq!(jit.data_addr("aligned").unwrap() as usize % 64, 0); + } +} diff --git a/src/target/src/regalloc/addrfold.rs b/src/target/src/regalloc/addrfold.rs index f0bd582..fafd96f 100644 --- a/src/target/src/regalloc/addrfold.rs +++ b/src/target/src/regalloc/addrfold.rs @@ -211,10 +211,7 @@ mod tests { use volt_ir::types::{IntDesc, TypeKind}; fn i32_ty(func: &mut Function) -> volt_ir::types::Type { - func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })) + func.types.intern(TypeKind::Int(IntDesc { bits: 32 })) } fn fold_in_range(func: &F, mem_inst: Inst) -> Option { @@ -251,7 +248,7 @@ mod tests { i32t, Opcode::Load(volt_ir::function::Load { ptr: p, - volatile: false, + mem: volt_ir::function::MemFlags::new(), }), ); func.set_terminator(b, Terminator::Ret(Ret::one(load))); @@ -280,7 +277,7 @@ mod tests { i32t, Opcode::Load(volt_ir::function::Load { ptr: base, - volatile: false, + mem: volt_ir::function::MemFlags::new(), }), ); func.set_terminator(b, Terminator::Ret(Ret::one(load))); @@ -312,7 +309,7 @@ mod tests { i32t, Opcode::Load(volt_ir::function::Load { ptr: p, - volatile: false, + mem: volt_ir::function::MemFlags::new(), }), ); func.set_terminator(b, Terminator::Ret(Ret::one(load))); @@ -344,7 +341,7 @@ mod tests { i32t, Opcode::Load(volt_ir::function::Load { ptr: p, - volatile: false, + mem: volt_ir::function::MemFlags::new(), }), ); let _ = load; diff --git a/src/target/src/regalloc/ir.rs b/src/target/src/regalloc/ir.rs index b9c9c8d..dd1a81d 100644 --- a/src/target/src/regalloc/ir.rs +++ b/src/target/src/regalloc/ir.rs @@ -143,6 +143,11 @@ pub fn visit_inst_operands(func: &F, inst: Inst, f: &mut impl FnMut(Val f(rd, false); } } + Opcode::Intrinsic(n) => { + for a in func.value_list(n.args) { + f(*a, false); + } + } Opcode::If(cf) => { f(cf.cond, false); for a in func.block_args(cf.then) { diff --git a/src/target/src/regalloc/wimmer.rs b/src/target/src/regalloc/wimmer.rs index 519f955..54662b7 100644 --- a/src/target/src/regalloc/wimmer.rs +++ b/src/target/src/regalloc/wimmer.rs @@ -3697,10 +3697,7 @@ mod tests { } fn i32_ty(func: &mut Function) -> volt_ir::types::Type { - func.types.intern(TypeKind::Int(IntDesc { - signed: true, - bits: 32, - })) + func.types.intern(TypeKind::Int(IntDesc { bits: 32 })) } #[test] diff --git a/src/target/src/x86_64/cases.rs b/src/target/src/x86_64/cases.rs new file mode 100644 index 0000000..dbf78cd --- /dev/null +++ b/src/target/src/x86_64/cases.rs @@ -0,0 +1,676 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Execution cases for the operations whose result depends on signedness. Each [`Case`] is one +//! function with the arguments to call it with and the value a correct execution returns; the +//! expected values come from the plain Rust reference arithmetic below, written independently +//! of the instruction selector. `tests.rs` executes the cases natively; `codegen_tests.rs` +//! compiles them on every host and checks the machine code decodes. +//! +//! Integer registers hold values narrower than 64 bits zero-extended, so a correct function +//! also returns them zero-extended, and ignores whatever the caller left above the declared +//! width of an argument. The cases pin both: arguments carry dirty upper bits on purpose and +//! results are compared exactly. +use alloc::{format, string::String, vec, vec::Vec}; +use volt_ir::function::*; + +/// An argument or return value. +#[derive(Clone, Copy, Debug)] +pub enum Val { + Int(u64), + F32(f32), + F64(f64), +} + +impl Val { + /// The raw register contents (an `f32` in the low 32 bits). + pub fn bits(self) -> u64 { + match self { + Val::Int(v) => v, + Val::F32(v) => v.to_bits() as u64, + Val::F64(v) => v.to_bits(), + } + } +} + +pub struct Case { + pub name: String, + pub function: Function, + /// `(arguments, expected result)` pairs. + pub runs: Vec<(Vec, Val)>, +} + +pub fn mask(bits: u32) -> u64 { + if bits >= 64 { + u64::MAX + } else { + (1 << bits) - 1 + } +} + +/// `v` read as a two's complement integer of `bits` bits. +pub fn sext(v: u64, bits: u32) -> i64 { + let s = 64 - bits; + ((v << s) as i64) >> s +} + +/// `a` with the bits above `bits` filled with garbage when `dirty` is odd. +fn dirty(a: u64, bits: u32, k: usize) -> u64 { + if k & 1 == 1 { a | !mask(bits) } else { a } +} + +fn build( + params: &[&str], + ret: &str, + body: impl FnOnce(&mut Function, Block, &[Value], volt_ir::types::Type) -> Value, +) -> Function { + let mut f = Function::new(); + let b = f.append_block(); + let args: Vec = params + .iter() + .map(|p| { + let t = f.types.parse_type(p).unwrap(); + f.append_block_param(b, t) + }) + .collect(); + let t = f.types.parse_type(ret).unwrap(); + let v = body(&mut f, b, &args, t); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + f +} + +pub fn arith(kind: &str, op: BinOp) -> Function { + build(&[kind, kind], kind, |f, b, a, t| { + f.append_inst( + b, + t, + Opcode::Arith(Arith { + op, + lhs: a[0], + rhs: a[1], + }), + ) + }) +} + +pub fn arith_imm(kind: &str, op: BinOp, imm: i64) -> Function { + build(&[kind], kind, |f, b, a, t| { + f.append_inst(b, t, Opcode::ArithImm(ArithImm { op, lhs: a[0], imm })) + }) +} + +pub fn compare(kind: &str, op: CmpOp) -> Function { + build(&[kind, kind], "bool", |f, b, a, t| { + f.append_inst( + b, + t, + Opcode::Icmp(Compare { + op, + lhs: a[0], + rhs: a[1], + }), + ) + }) +} + +pub fn convert(from: &str, to: &str, kind: ConvertKind) -> Function { + build(&[from], to, |f, b, a, t| { + f.append_inst(b, t, Opcode::Convert(Convert { value: a[0], kind })) + }) +} + +/// The value of `op` on two `bits`-bit integers, or `None` where the hardware traps (a zero +/// divisor, or the most negative value divided by -1 at the 32 and 64 bit widths). +pub fn arith_reference(op: BinOp, bits: u32, a: u64, b: u64) -> Option { + let (a, b) = (a & mask(bits), b & mask(bits)); + let (sa, sb) = (sext(a, bits), sext(b, bits)); + let trapping_signed = (bits == 32 || bits == 64) && sa == i64::MIN >> (64 - bits) && sb == -1; + let r = match op { + BinOp::Add => a.wrapping_add(b), + BinOp::Sub => a.wrapping_sub(b), + BinOp::Mul => a.wrapping_mul(b), + BinOp::BitAnd => a & b, + BinOp::BitOr => a | b, + BinOp::BitXor => a ^ b, + BinOp::Shl => a << b, + BinOp::Shr => a >> b, + BinOp::Sar => (sa >> b) as u64, + BinOp::UDiv if b != 0 => a / b, + BinOp::URem if b != 0 => a % b, + BinOp::SDiv if b != 0 && !trapping_signed => sa.wrapping_div(sb) as u64, + BinOp::SRem if b != 0 && !trapping_signed => sa.wrapping_rem(sb) as u64, + BinOp::UMulh => ((a as u128 * b as u128) >> bits) as u64, + BinOp::SMulh => ((sa as i128 * sb as i128) >> bits) as u64, + _ => return None, + }; + Some(r & mask(bits)) +} + +pub const WIDTHS: [u32; 6] = [8, 16, 32, 33, 48, 64]; + +pub fn samples(bits: u32) -> Vec { + let m = mask(bits); + let mut v = vec![ + 0, + 1, + 2, + 3, + 5, + 7, + 100, + 200, + m, + m - 1, + m >> 1, + (m >> 1) + 1, + ((m >> 1) + 1) | 1, + 0x5555_5555_5555_5555 & m, + 0xaaaa_aaaa_aaaa_aaaa & m, + 0x0123_4567_89ab_cdef & m, + ]; + for x in &mut v { + *x &= m; + } + v.sort_unstable(); + v.dedup(); + v +} + +fn is_shift(op: BinOp) -> bool { + matches!(op, BinOp::Shl | BinOp::Shr | BinOp::Sar) +} + +/// Constants: the payload is any `i64`, and the register must hold its low `bits` +/// zero-extended, whatever the sign of the payload. +fn iconst_cases(out: &mut Vec) { + for bits in WIDTHS { + let kind = format!("i{bits}"); + for n in [ + -1i64, + -2, + 1, + -100, + 127, + -128, + 255, + 0x1234_5678_9abc_def0, + i64::MIN, + ] { + out.push(Case { + name: format!("{kind} Iconst {n}"), + function: build(&[], &kind, |f, b, _, t| { + f.append_inst(b, t, Opcode::Iconst(n)) + }), + runs: vec![(vec![], Val::Int(n as u64 & mask(bits)))], + }); + } + } +} + +fn arith_cases(out: &mut Vec) { + for bits in WIDTHS { + let kind = format!("i{bits}"); + for op in [ + BinOp::Add, + BinOp::Sub, + BinOp::Mul, + BinOp::BitAnd, + BinOp::BitOr, + BinOp::BitXor, + BinOp::Shl, + BinOp::Shr, + BinOp::Sar, + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::UMulh, + BinOp::SMulh, + ] { + let mut runs = Vec::new(); + let mut k = 0; + for &a in &samples(bits) { + for &b in &samples(bits) { + // A shift amount is only meaningful below the width. + let b = if is_shift(op) { b % bits as u64 } else { b }; + if let Some(want) = arith_reference(op, bits, a, b) { + k += 1; + runs.push(( + vec![Val::Int(dirty(a, bits, k)), Val::Int(dirty(b, bits, k / 2))], + Val::Int(want), + )); + } + } + } + out.push(Case { + name: format!("{kind} {op:?}"), + function: arith(&kind, op), + runs, + }); + } + } +} + +/// `op` against a constant: the immediate is the value an `Iconst` of the operand type would +/// hold, so a negative immediate on a narrow type means its unsigned reading. +fn immediate_cases(out: &mut Vec) { + for (bits, op, imm) in [ + (8, BinOp::Add, -1), + (8, BinOp::UDiv, -56), + (8, BinOp::URem, -56), + (8, BinOp::SDiv, -3), + (8, BinOp::SRem, -3), + (8, BinOp::Shl, 3), + (8, BinOp::Shr, 3), + (8, BinOp::Sar, 3), + (16, BinOp::UDiv, -1000), + (16, BinOp::SRem, -1000), + (16, BinOp::Sar, 15), + (32, BinOp::UDiv, -3), + (32, BinOp::SDiv, -3), + (32, BinOp::SRem, -3), + (32, BinOp::Sar, 31), + (32, BinOp::Shr, 31), + (48, BinOp::UDiv, -3), + (48, BinOp::SDiv, -3), + (48, BinOp::Sar, 47), + (64, BinOp::SDiv, -3), + (64, BinOp::UDiv, -3), + (64, BinOp::Sar, 63), + (64, BinOp::Shr, 63), + ] { + let kind = format!("i{bits}"); + let mut runs = Vec::new(); + for (k, &a) in samples(bits).iter().enumerate() { + if let Some(want) = arith_reference(op, bits, a, imm as u64) { + runs.push((vec![Val::Int(dirty(a, bits, k))], Val::Int(want))); + } + } + out.push(Case { + name: format!("{kind} {op:?} imm {imm}"), + function: arith_imm(&kind, op, imm), + runs, + }); + } +} + +pub fn compare_reference(op: CmpOp, bits: u32, a: u64, b: u64) -> bool { + let (a, b) = (a & mask(bits), b & mask(bits)); + let (sa, sb) = (sext(a, bits), sext(b, bits)); + match op { + CmpOp::Eq => a == b, + CmpOp::Ne => a != b, + CmpOp::Slt => sa < sb, + CmpOp::Sle => sa <= sb, + CmpOp::Sgt => sa > sb, + CmpOp::Sge => sa >= sb, + CmpOp::Ult => a < b, + CmpOp::Ule => a <= b, + CmpOp::Ugt => a > b, + CmpOp::Uge => a >= b, + CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge => unreachable!("float compare"), + } +} + +fn compare_cases(out: &mut Vec) { + let signed = [CmpOp::Slt, CmpOp::Sle, CmpOp::Sgt, CmpOp::Sge]; + let unsigned = [CmpOp::Ult, CmpOp::Ule, CmpOp::Ugt, CmpOp::Uge]; + let equality = [CmpOp::Eq, CmpOp::Ne]; + // `ptr` is an unsigned 64-bit quantity: it has no signed order. + for kind in WIDTHS + .map(|b| format!("i{b}")) + .into_iter() + .chain(["ptr".into()]) + { + let bits = kind[1..].parse().unwrap_or(64); + let ops: Vec = if kind == "ptr" { + equality.iter().chain(&unsigned).copied().collect() + } else { + equality + .iter() + .chain(&signed) + .chain(&unsigned) + .copied() + .collect() + }; + for op in ops { + let mut runs = Vec::new(); + let mut k = 0; + for &a in &samples(bits) { + for &b in &samples(bits) { + k += 1; + runs.push(( + vec![Val::Int(dirty(a, bits, k)), Val::Int(dirty(b, bits, k / 3))], + Val::Int(compare_reference(op, bits, a, b) as u64), + )); + } + } + out.push(Case { + name: format!("{kind} {op:?}"), + function: compare(&kind, op), + runs, + }); + } + } +} + +fn integer_convert_cases(out: &mut Vec) { + // (from, to) widths of the integer width changes. + let trunc = [ + (64, 32), + (64, 8), + (64, 16), + (64, 48), + (48, 8), + (33, 8), + (32, 16), + (32, 8), + (16, 8), + ]; + let ext = [ + (8, 16), + (8, 32), + (8, 48), + (8, 64), + (16, 32), + (16, 64), + (32, 33), + (32, 64), + (33, 64), + (48, 64), + ]; + for (kind, pairs) in [ + (ConvertKind::Trunc, &trunc[..]), + (ConvertKind::Zext, &ext[..]), + (ConvertKind::Sext, &ext[..]), + ] { + for &(from, to) in pairs { + let mut runs = Vec::new(); + for (k, &a) in samples(from).iter().enumerate() { + let want = match kind { + ConvertKind::Trunc => a & mask(to), + ConvertKind::Zext => a, + _ => sext(a, from) as u64 & mask(to), + }; + runs.push((vec![Val::Int(dirty(a, from, k))], Val::Int(want))); + } + out.push(Case { + name: format!("i{from} {kind:?} i{to}"), + function: convert(&format!("i{from}"), &format!("i{to}"), kind), + runs, + }); + } + } + for to in [32, 64] { + out.push(Case { + name: format!("bool Zext i{to}"), + function: convert("bool", &format!("i{to}"), ConvertKind::Zext), + runs: vec![ + (vec![Val::Int(0)], Val::Int(0)), + (vec![Val::Int(1)], Val::Int(1)), + // Only the declared bit of the argument counts. + (vec![Val::Int(0xffff_ff00)], Val::Int(0)), + (vec![Val::Int(0xffff_ff01)], Val::Int(1)), + ], + }); + } +} + +fn float_convert_cases(out: &mut Vec) { + for bits in [8, 16, 32, 48, 64] { + for (fkind, double) in [("f64", true), ("f32", false)] { + for kind in [ConvertKind::SiToFp, ConvertKind::UiToFp] { + let mut runs = Vec::new(); + for (k, &a) in samples(bits).iter().enumerate() { + let want = match (kind == ConvertKind::SiToFp, double) { + (true, true) => Val::F64(sext(a, bits) as f64), + (true, false) => Val::F32(sext(a, bits) as f32), + (false, true) => Val::F64(a as f64), + (false, false) => Val::F32(a as f32), + }; + runs.push((vec![Val::Int(dirty(a, bits, k))], want)); + } + out.push(Case { + name: format!("i{bits} {kind:?} {fkind}"), + function: convert(&format!("i{bits}"), fkind, kind), + runs, + }); + } + } + } + // In-range sources only: out-of-range results are the hardware's "integer indefinite". + let signed_sources = |bits: u32| -> Vec { + match bits { + 8 => vec![0.0, 1.5, -1.5, 100.9, -100.9, 127.0, -128.0], + 16 => vec![0.0, 1.5, -1.5, 30000.7, -30000.7, 32767.0, -32768.0], + 32 => vec![0.0, 1.5, -1.5, 2.0e9, -2.0e9, 2147483520.0, -2147483648.0], + _ => vec![ + 0.0, + 1.5, + -1.5, + 1.0e15, + -1.0e15, + 9.0e18, + -9.0e18, + -9223372036854775808.0, + ], + } + }; + let unsigned_sources = |bits: u32| -> Vec { + match bits { + 8 => vec![0.0, 1.9, 200.5, 255.0], + 16 => vec![0.0, 1.9, 40000.5, 65535.0], + 32 => vec![0.0, 0.5, 3.0e9, 4294967295.0], + _ => vec![ + 0.0, + 1.5, + 9.0e18, + 9223372036854775807.0, + 9223372036854775808.0, + 1.8e19, + 18446744073709549568.0, + ], + } + }; + for bits in [8, 16, 32, 64] { + for (fkind, double) in [("f64", true), ("f32", false)] { + for kind in [ConvertKind::FpToSi, ConvertKind::FpToUi] { + let sources = if kind == ConvertKind::FpToSi { + signed_sources(bits) + } else { + unsigned_sources(bits) + }; + let mut runs = Vec::new(); + for v in sources { + // Round the source to the source type first; the reference then sees + // exactly the value the function is given. + let (arg, truncated) = if double { + (Val::F64(v), v) + } else { + (Val::F32(v as f32), v as f32 as f64) + }; + let half = 2f64.powi(bits as i32 - 1); + let in_range = if kind == ConvertKind::FpToSi { + truncated >= -half && truncated < half + } else { + truncated > -1.0 && truncated < 2.0 * half + }; + if !in_range { + continue; + } + let want = if kind == ConvertKind::FpToSi { + truncated as i64 as u64 & mask(bits) + } else { + truncated as u64 & mask(bits) + }; + runs.push((vec![arg], Val::Int(want))); + } + out.push(Case { + name: format!("{fkind} {kind:?} i{bits}"), + function: convert(fkind, &format!("i{bits}"), kind), + runs, + }); + } + } + } + out.push(Case { + name: "f32 FpResize f64".into(), + function: convert("f32", "f64", ConvertKind::FpResize), + runs: [0.0f32, 1.5, -2.25, 1.0e10, 3.0e38, 1.0e-30] + .map(|v| (vec![Val::F32(v)], Val::F64(v as f64))) + .into(), + }); + out.push(Case { + name: "f64 FpResize f32".into(), + function: convert("f64", "f32", ConvertKind::FpResize), + runs: [0.0f64, 1.5, -2.25, 1.0e10, 3.0e38, 0.1] + .map(|v| (vec![Val::F64(v)], Val::F32(v as f32))) + .into(), + }); +} + +/// `<4 x i32>` lanes: `lhs = [a0, a1, a1, a0]`, `rhs = [b0, b1, b0, b1]`, and the function +/// returns lane 0 | lane 2 << 32 of `lhs rhs`. +fn vector_function(op: Result) -> Function { + build(&["i32", "i32", "i32", "i32"], "i64", |f, b, p, wide| { + let vector = f.types.parse_type("<4 x i32>").unwrap(); + let int = f.types.parse_type("i32").unwrap(); + let fields = f.intern_values(&[p[0], p[1], p[1], p[0]]); + let lhs = f.append_inst(b, vector, Opcode::StructNew(StructNew { fields })); + let fields = f.intern_values(&[p[2], p[3], p[2], p[3]]); + let rhs = f.append_inst(b, vector, Opcode::StructNew(StructNew { fields })); + let v = match op { + Ok(op) => f.append_inst(b, vector, Opcode::Arith(Arith { op, lhs, rhs })), + Err(op) => f.append_inst(b, vector, Opcode::Icmp(Compare { op, lhs, rhs })), + }; + let lane = |f: &mut Function, index| { + let x = f.append_inst( + b, + int, + Opcode::Extract(Extract { + aggregate: v, + index, + }), + ); + f.append_inst( + b, + wide, + Opcode::Convert(Convert { + value: x, + kind: ConvertKind::Zext, + }), + ) + }; + let low = lane(f, 0); + let high = lane(f, 2); + let high = f.append_inst( + b, + wide, + Opcode::ArithImm(ArithImm { + op: BinOp::Shl, + lhs: high, + imm: 32, + }), + ); + f.append_inst( + b, + wide, + Opcode::Arith(Arith { + op: BinOp::BitOr, + lhs: low, + rhs: high, + }), + ) + }) +} + +fn vector_cases(out: &mut Vec) { + let lanes: [u64; 6] = [0, 1, 7, 0x7fff_ffff, 0x8000_0000, 0xffff_ffff]; + let combos = || { + lanes.into_iter().flat_map(move |a0| { + lanes.into_iter().flat_map(move |a1| { + lanes + .into_iter() + .flat_map(move |b0| lanes.into_iter().map(move |b1| [a0, a1, b0, b1])) + }) + }) + }; + for op in [ + BinOp::Add, + BinOp::Mul, + BinOp::Shr, + BinOp::Sar, + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::UMulh, + BinOp::SMulh, + ] { + let mut runs = Vec::new(); + for [a0, a1, b0, b1] in combos() { + let (b0, b1) = if is_shift(op) { + (b0 % 32, b1 % 32) + } else { + (b0, b1) + }; + // Every lane executes, so a lane that would trap rules the whole input out. + let lanes = [(a0, b0), (a1, b1), (a1, b0), (a0, b1)] + .map(|(a, b)| arith_reference(op, 32, a, b)); + if let [Some(l0), Some(_), Some(l2), Some(_)] = lanes { + runs.push(( + [a0, a1, b0, b1].map(Val::Int).into(), + Val::Int(l0 | l2 << 32), + )); + } + } + out.push(Case { + name: format!("<4 x i32> {op:?}"), + function: vector_function(Ok(op)), + runs, + }); + } + for op in [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Slt, + CmpOp::Sle, + CmpOp::Sgt, + CmpOp::Sge, + CmpOp::Ult, + CmpOp::Ule, + CmpOp::Ugt, + CmpOp::Uge, + ] { + let lane = |a, b| { + if compare_reference(op, 32, a, b) { + 0xffff_ffffu64 + } else { + 0 + } + }; + let runs = combos() + .map(|[a0, a1, b0, b1]| { + ( + [a0, a1, b0, b1].map(Val::Int).into(), + Val::Int(lane(a0, b0) | lane(a1, b0) << 32), + ) + }) + .collect(); + out.push(Case { + name: format!("<4 x i32> {op:?}"), + function: vector_function(Err(op)), + runs, + }); + } +} + +pub fn all() -> Vec { + let mut out = Vec::new(); + arith_cases(&mut out); + iconst_cases(&mut out); + immediate_cases(&mut out); + compare_cases(&mut out); + integer_convert_cases(&mut out); + float_convert_cases(&mut out); + vector_cases(&mut out); + out +} diff --git a/src/target/src/x86_64/codegen_tests.rs b/src/target/src/x86_64/codegen_tests.rs new file mode 100644 index 0000000..80dc398 --- /dev/null +++ b/src/target/src/x86_64/codegen_tests.rs @@ -0,0 +1,556 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Byte-level checks that signedness is chosen by the operation: the selector emits the signed +//! or the unsigned instruction (and the sign extension a signed operation needs) according to +//! the operator, never according to the type. They only compile code, so unlike `tests.rs` +//! (which executes it) they run on every host; the instruction text comes from this crate's +//! own disassembler, whose encodings are separately checked against llvm-mc. +use super::{Error, cases, compile, disasm}; +use alloc::{format, string::String, vec::Vec}; +use volt_ir::function::*; + +/// The disassembled instructions of `f`, one string per instruction. +fn asm(f: &Function) -> Vec { + let code = compile(f).expect("compiles"); + disasm::format(&code) + .lines() + .map(|l| l.split_once(": ").expect("offset prefix").1.into()) + .collect() +} + +fn has(lines: &[String], text: &str) -> bool { + lines.iter().any(|l| l == text) +} + +#[track_caller] +fn expect(lines: &[String], present: &[&str], absent: &[&str]) { + for p in present { + assert!(has(lines, p), "missing `{p}` in:\n{}", lines.join("\n")); + } + for a in absent { + assert!(!has(lines, a), "unexpected `{a}` in:\n{}", lines.join("\n")); + } +} + +#[test] +fn every_case_compiles_to_decodable_code() { + for case in cases::all() { + let code = compile(&case.function).unwrap_or_else(|e| panic!("{}: {e}", case.name)); + let text = disasm::format(&code); + assert!(!text.contains(".byte"), "{}:\n{text}", case.name); + check_case_shape(&case); + } +} + +/// The arguments and expected results of a case match the function's signature, and an +/// expected integer result is already zero-extended from its declared width. +fn check_case_shape(case: &cases::Case) { + use cases::Val; + use volt_ir::types::{FloatKind, TypeKind}; + let f = &case.function; + let class = |v: Value| match f.types.type_kind(f.value_type(v)) { + TypeKind::Float(FloatKind::F32) => "f32", + TypeKind::Float(FloatKind::F64) => "f64", + _ => "int", + }; + let params = f.block_params(Block(0)); + let Some(Terminator::Ret(ret)) = f.terminator(Block(0)) else { + panic!("{}: no return", case.name); + }; + let result = ret.slice()[0]; + let bits = match f.types.type_kind(f.value_type(result)) { + TypeKind::Int(i) => i.bits as u32, + TypeKind::Bool => 1, + _ => 64, + }; + assert!(!case.runs.is_empty(), "{}", case.name); + for (args, want) in &case.runs { + assert_eq!(args.len(), params.len(), "{}", case.name); + for (a, &p) in args.iter().zip(params) { + let want_class = match a { + Val::Int(_) => "int", + Val::F32(_) => "f32", + Val::F64(_) => "f64", + }; + assert_eq!(want_class, class(p), "{}", case.name); + } + let result_class = match want { + Val::Int(_) => "int", + Val::F32(_) => "f32", + Val::F64(_) => "f64", + }; + assert_eq!(result_class, class(result), "{}", case.name); + if let Val::Int(_) = want { + assert_eq!( + want.bits() & !cases::mask(bits), + 0, + "{} {args:x?}", + case.name + ); + } + } +} + +#[test] +fn division_follows_the_operator() { + for (kind, wide) in [("i32", false), ("i64", true)] { + let r11 = if wide { "r11" } else { "r11d" }; + let unsigned = asm(&cases::arith(kind, BinOp::UDiv)); + expect( + &unsigned, + &["xor edx, edx", &format!("div {r11}")], + &["cdq", "cqo", &format!("idiv {r11}")], + ); + let signed = asm(&cases::arith(kind, BinOp::SDiv)); + expect( + &signed, + &[if wide { "cqo" } else { "cdq" }, &format!("idiv {r11}")], + &["xor edx, edx", &format!("div {r11}")], + ); + // The remainder is the same divide read from rdx. + let urem = asm(&cases::arith(kind, BinOp::URem)); + let srem = asm(&cases::arith(kind, BinOp::SRem)); + expect(&urem, &[&format!("div {r11}"), "mov r10, rdx"], &[]); + expect(&srem, &[&format!("idiv {r11}"), "mov r10, rdx"], &[]); + } +} + +#[test] +fn narrow_signed_operations_sign_extend_their_operands_and_unsigned_ones_do_not() { + let sign_extension = ["shl r10, 56", "sar r10, 56", "shl r11, 56", "sar r11, 56"]; + for op in [BinOp::SDiv, BinOp::SRem, BinOp::SMulh] { + let lines = asm(&cases::arith("i8", op)); + expect(&lines, &sign_extension, &[]); + } + for op in [BinOp::UDiv, BinOp::URem, BinOp::UMulh] { + let lines = asm(&cases::arith("i8", op)); + expect(&lines, &[], &["sar r10, 56", "sar r11, 56"]); + } + // An arithmetic shift extends only the value shifted; a logical one extends nothing. + let sar = asm(&cases::arith("i8", BinOp::Sar)); + expect( + &sar, + &["shl r10, 56", "sar r10, 56", "sar r10d, cl"], + &["sar r11, 56"], + ); + let shr = asm(&cases::arith("i8", BinOp::Shr)); + expect(&shr, &["shr r10d, cl"], &["sar r10, 56", "sar r10d, cl"]); + // Wrapping operations need neither. + for op in [BinOp::Add, BinOp::Mul, BinOp::BitAnd] { + let lines = asm(&cases::arith("i8", op)); + expect(&lines, &[], &["sar r10, 56"]); + } +} + +#[test] +fn shifts_follow_the_operator() { + for (kind, r10) in [("i32", "r10d"), ("i64", "r10")] { + expect( + &asm(&cases::arith(kind, BinOp::Shl)), + &[&format!("shl {r10}, cl")], + &[&format!("shr {r10}, cl"), &format!("sar {r10}, cl")], + ); + expect( + &asm(&cases::arith(kind, BinOp::Shr)), + &[&format!("shr {r10}, cl")], + &[&format!("shl {r10}, cl"), &format!("sar {r10}, cl")], + ); + expect( + &asm(&cases::arith(kind, BinOp::Sar)), + &[&format!("sar {r10}, cl")], + &[&format!("shl {r10}, cl"), &format!("shr {r10}, cl")], + ); + } +} + +#[test] +fn high_product_follows_the_operator() { + expect( + &asm(&cases::arith("i32", BinOp::UMulh)), + &["mul r11d"], + &["imul r11d"], + ); + expect( + &asm(&cases::arith("i32", BinOp::SMulh)), + &["imul r11d"], + &["mul r11d"], + ); + expect( + &asm(&cases::arith("i64", BinOp::UMulh)), + &["mul r11"], + &["imul r11"], + ); + expect( + &asm(&cases::arith("i64", BinOp::SMulh)), + &["imul r11"], + &["mul r11"], + ); + // Narrower than 32 bits: a 64-bit product shifted down logically or arithmetically. + expect( + &asm(&cases::arith("i16", BinOp::UMulh)), + &["imul r10, r11", "shr r10, 16"], + &["sar r10, 16"], + ); + expect( + &asm(&cases::arith("i16", BinOp::SMulh)), + &["imul r10, r11", "sar r10, 16"], + &["shr r10, 16"], + ); + // Between 32 and 64 bits the double-width product spans rdx:rax. + expect( + &asm(&cases::arith("i48", BinOp::UMulh)), + &["mul r11"], + &["imul r11"], + ); + expect( + &asm(&cases::arith("i48", BinOp::SMulh)), + &["imul r11"], + &["mul r11"], + ); +} + +#[test] +fn comparisons_pick_the_condition_code_by_operator() { + // (operator, condition suffix of the set instruction) + for (op, cc) in [ + (CmpOp::Eq, "e"), + (CmpOp::Ne, "ne"), + (CmpOp::Slt, "l"), + (CmpOp::Sle, "le"), + (CmpOp::Sgt, "g"), + (CmpOp::Sge, "ge"), + (CmpOp::Ult, "b"), + (CmpOp::Ule, "be"), + (CmpOp::Ugt, "a"), + (CmpOp::Uge, "ae"), + ] { + for (kind, cmp) in [("i32", "cmp r10d, r11d"), ("i64", "cmp r10, r11")] { + let lines = asm(&cases::compare(kind, op)); + expect(&lines, &[cmp, &format!("set{cc} r10b")], &[]); + } + } + // Pointers compare unsigned. + expect( + &asm(&cases::compare("ptr", CmpOp::Ult)), + &["cmp r10, r11", "setb r10b"], + &["setl r10b"], + ); +} + +#[test] +fn narrow_signed_comparisons_sign_extend_both_operands() { + for op in [CmpOp::Slt, CmpOp::Sle, CmpOp::Sgt, CmpOp::Sge] { + expect( + &asm(&cases::compare("i8", op)), + &["shl r10, 56", "sar r10, 56", "shl r11, 56", "sar r11, 56"], + &[], + ); + expect( + &asm(&cases::compare("i48", op)), + &["shl r10, 16", "sar r10, 16", "cmp r10, r11"], + &[], + ); + } + for op in [ + CmpOp::Eq, + CmpOp::Ne, + CmpOp::Ult, + CmpOp::Ule, + CmpOp::Ugt, + CmpOp::Uge, + ] { + expect( + &asm(&cases::compare("i8", op)), + &[], + &["sar r10, 56", "sar r11, 56"], + ); + } +} + +#[test] +fn integer_width_changes_pick_the_extension_by_conversion() { + use ConvertKind::*; + // 32 -> 64: a sign extension is `movsxd`; a zero extension needs no instruction at all, + // because a 32-bit value is already held zero-extended. + let sext = asm(&cases::convert("i32", "i64", Sext)); + let zext = asm(&cases::convert("i32", "i64", Zext)); + expect(&sext, &["movsxd r10, r10d"], &[]); + expect(&zext, &[], &["movsxd r10, r10d"]); + assert!(zext.len() < sext.len()); + // Narrower sources shift the sign (or zero) in. + expect( + &asm(&cases::convert("i8", "i64", Sext)), + &["shl r10, 56", "sar r10, 56"], + &[], + ); + expect( + &asm(&cases::convert("i8", "i64", Zext)), + &[], + &["sar r10, 56", "shr r10, 56"], + ); + // A sign extension to a narrower-than-64 destination is zero-extended afterwards. + expect( + &asm(&cases::convert("i8", "i32", Sext)), + &["shl r10, 56", "sar r10, 56", "mov r10d, r10d"], + &[], + ); + // Truncation keeps the low bits, zero-extended. + expect( + &asm(&cases::convert("i64", "i32", Trunc)), + &["mov r10d, r10d"], + &[], + ); + expect( + &asm(&cases::convert("i64", "i8", Trunc)), + &["shl r10, 56", "shr r10, 56"], + &["sar r10, 56"], + ); + expect( + &asm(&cases::convert("bool", "i64", Zext)), + &[], + &["sar r10, 56", "shr r10, 56"], + ); +} + +#[test] +fn integer_to_float_conversions_pick_the_signedness_by_conversion() { + use ConvertKind::*; + // 64-bit: the unsigned conversion needs the halve-and-double path for a set top bit. + let signed = asm(&cases::convert("i64", "f64", SiToFp)); + let unsigned = asm(&cases::convert("i64", "f64", UiToFp)); + expect(&signed, &["cvtsi2sd xmm13, r10"], &["test r10, r10"]); + expect( + &unsigned, + &["cvtsi2sd xmm13, r10", "test r10, r10", "addsd xmm13, xmm13"], + &[], + ); + expect( + &asm(&cases::convert("i64", "f32", UiToFp)), + &["cvtsi2ss xmm13, r10", "test r10, r10", "addss xmm13, xmm13"], + &[], + ); + // 32-bit: the signed conversion sign-extends first, the unsigned one converts the + // zero-extended value as a (non-negative) 64-bit integer. + let signed = asm(&cases::convert("i32", "f64", SiToFp)); + let unsigned = asm(&cases::convert("i32", "f64", UiToFp)); + expect(&signed, &["movsxd r10, r10d", "cvtsi2sd xmm13, r10"], &[]); + expect( + &unsigned, + &["cvtsi2sd xmm13, r10"], + &["movsxd r10, r10d", "test r10, r10"], + ); + // Narrow: sign-extended by shifts for the signed conversion only. + expect( + &asm(&cases::convert("i8", "f64", SiToFp)), + &["shl r10, 56", "sar r10, 56"], + &[], + ); + expect( + &asm(&cases::convert("i8", "f64", UiToFp)), + &["cvtsi2sd xmm13, r10"], + &["sar r10, 56"], + ); +} + +#[test] +fn float_to_integer_conversions_pick_the_signedness_by_conversion() { + use ConvertKind::*; + // 64-bit unsigned results above 2^63 take the subtract-and-flip path. + let signed = asm(&cases::convert("f64", "i64", FpToSi)); + let unsigned = asm(&cases::convert("f64", "i64", FpToUi)); + expect( + &signed, + &["cvttsd2si r10, xmm13"], + &["ucomisd xmm13, xmm14"], + ); + expect( + &unsigned, + &[ + "cvttsd2si r10, xmm13", + "ucomisd xmm13, xmm14", + "subsd xmm13, xmm14", + ], + &[], + ); + expect( + &asm(&cases::convert("f32", "i64", FpToUi)), + &[ + "cvttss2si r10, xmm13", + "ucomiss xmm13, xmm14", + "subss xmm13, xmm14", + ], + &[], + ); + // A signed 32-bit result is a 32-bit truncating convert; an unsigned one is a 64-bit + // convert (so values past 2^31 survive) whose low 32 bits are kept. + expect( + &asm(&cases::convert("f64", "i32", FpToSi)), + &["cvttsd2si r10d, xmm13", "mov r10d, r10d"], + &["cvttsd2si r10, xmm13"], + ); + expect( + &asm(&cases::convert("f64", "i32", FpToUi)), + &["cvttsd2si r10, xmm13", "mov r10d, r10d"], + &["cvttsd2si r10d, xmm13"], + ); +} + +#[test] +fn float_resize_converts_between_widths() { + use ConvertKind::*; + expect( + &asm(&cases::convert("f32", "f64", FpResize)), + &["cvtss2sd xmm13, xmm13"], + &["cvtsd2ss xmm13, xmm13"], + ); + expect( + &asm(&cases::convert("f64", "f32", FpResize)), + &["cvtsd2ss xmm13, xmm13"], + &["cvtss2sd xmm13, xmm13"], + ); +} + +#[test] +fn loads_zero_extend_and_never_sign_extend() { + for (kind, load) in [ + ("i8", "movzx r10, byte ptr [r11]"), + ("i16", "movzx r10, word ptr [r11]"), + ("i32", "mov r10d, dword ptr [r11]"), + ("i64", "mov r10, qword ptr [r11]"), + ] { + let mut f = Function::new(); + let p = f.types.parse_type("ptr").unwrap(); + let t = f.types.parse_type(kind).unwrap(); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let v = f.append_load(b, t, ptr, MemFlags::new()); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let lines = asm(&f); + expect(&lines, &[load], &[]); + assert!( + lines.iter().all(|l| !l.starts_with("movsx")), + "{kind}:\n{}", + lines.join("\n") + ); + } +} + +#[test] +fn narrow_arguments_and_call_results_are_zero_extended() { + // An argument is reloaded from its home slot at its declared width, which discards + // whatever the caller left above it. + let lines = asm(&cases::arith("i8", BinOp::Add)); + let loads = lines + .iter() + .filter(|l| l.starts_with("movzx r10, byte ptr [rbp - ")) + .count(); + assert_eq!(loads, 2, "{lines:?}"); + // A call result is cleaned before use: the ABI leaves the upper bits undefined. + let mut f = Function::new(); + let byte = f.types.parse_type("i8").unwrap(); + let p = f.types.parse_type("ptr").unwrap(); + let b = f.append_block(); + let target = f.append_block_param(b, p); + let args = f.intern_values(&[]); + let r = f.append_inst( + b, + byte, + Opcode::CallIndirect(CallIndirect { + target, + args, + is_variadic: false, + num_fixed: 0, + ret_dest: None, + ret_regs: 0, + ret_pieces: [RetPiece::default(); 4], + sret: false, + }), + ); + f.set_terminator(b, Terminator::Ret(Ret::one(r))); + expect(&asm(&f), &["shl rax, 56", "shr rax, 56"], &[]); +} + +#[test] +fn immediates_are_truncated_to_the_operand_width() { + // `udiv i8 x, -56` divides by 200, not by 2^64 - 56. + let lines = asm(&cases::arith_imm("i8", BinOp::UDiv, -56)); + expect(&lines, &["mov r11, 200"], &[]); + let lines = asm(&cases::arith_imm("i32", BinOp::UDiv, -3)); + expect(&lines, &["mov r11, 4294967293"], &[]); + let lines = asm(&cases::arith_imm("i64", BinOp::UDiv, -3)); + expect(&lines, &["mov r11, 18446744073709551613"], &[]); +} + +fn single(kind: &str, op: impl FnOnce(Value, Value) -> Opcode, ret: &str) -> Function { + let mut f = Function::new(); + let t = f.types.parse_type(kind).unwrap(); + let r = f.types.parse_type(ret).unwrap(); + let b = f.append_block(); + let x = f.append_block_param(b, t); + let y = f.append_block_param(b, t); + let v = f.append_inst(b, r, op(x, y)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + f +} + +#[test] +fn operations_applied_to_the_wrong_class_fail_closed() { + use Opcode::{Arith as A, Icmp}; + let arith = |op| move |lhs, rhs| A(Arith { op, lhs, rhs }); + let cmp = |op| move |lhs, rhs| Icmp(Compare { op, lhs, rhs }); + // The floating-point division on integers, the integer divisions on floats. + for op in [BinOp::Div, BinOp::Rem] { + let f = single("i32", arith(op), "i32"); + assert_eq!(compile(&f), Err(Error::InvalidFunction), "{op:?}"); + } + for op in [ + BinOp::UDiv, + BinOp::SDiv, + BinOp::URem, + BinOp::SRem, + BinOp::Shr, + BinOp::Sar, + ] { + let f = single("f64", arith(op), "f64"); + assert!(matches!(compile(&f), Err(Error::Unsupported(_))), "{op:?}"); + } + // The float orders on integers, the integer orders on floats. + for op in [CmpOp::Lt, CmpOp::Le, CmpOp::Gt, CmpOp::Ge] { + let f = single("i32", cmp(op), "bool"); + assert_eq!(compile(&f), Err(Error::InvalidFunction), "{op:?}"); + } + for op in [CmpOp::Slt, CmpOp::Ult, CmpOp::Sge, CmpOp::Uge] { + let f = single("f64", cmp(op), "bool"); + assert_eq!(compile(&f), Err(Error::InvalidFunction), "{op:?}"); + } + // An integer conversion of a float, a float conversion of the wrong class. + use ConvertKind::*; + for (from, to, kind) in [ + ("f64", "i64", Zext), + ("i32", "i64", FpResize), + ("i32", "f64", FpToSi), + ("f64", "f32", SiToFp), + ] { + let f = cases::convert(from, to, kind); + assert!(compile(&f).is_err(), "{from} {kind:?} {to}"); + } +} + +#[test] +fn integer_dot_products_are_not_lowered() { + let mut f = Function::new(); + let vector = f.types.parse_type("<4 x i32>").unwrap(); + let int = f.types.parse_type("i32").unwrap(); + let b = f.append_block(); + let acc = f.append_block_param(b, int); + let x = f.append_block_param(b, vector); + let y = f.append_block_param(b, vector); + for signed in [false, true] { + let mut f = f.clone_func(); + let v = f.append_dot(b, acc, x, y, signed); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + assert!( + matches!(compile(&f), Err(Error::Unsupported(_))), + "signed={signed}" + ); + } +} diff --git a/src/target/src/x86_64/disasm.rs b/src/target/src/x86_64/disasm.rs index b4d0903..fad8979 100644 --- a/src/target/src/x86_64/disasm.rs +++ b/src/target/src/x86_64/disasm.rs @@ -401,11 +401,15 @@ impl D<'_> { if op == 0xa8 { "al" } else { self.rv(0) }, self.imm(if op == 0xa8 { 1 } else { 4 })? ), - _ if (op <= 0x3b && op & 7 <= 3) || matches!(op, 0x84 | 0x85 | 0x88 | 0x89 | 0x8a) => { + _ if (op <= 0x3b && op & 7 <= 3) + || matches!(op, 0x84 | 0x85 | 0x86 | 0x87 | 0x88 | 0x89 | 0x8a) => + { let o = self.modrm()?; let byte = op & 1 == 0; let name = if op >= 0x88 { "mov" + } else if op >= 0x86 { + "xchg" } else if op >= 0x84 { "test" } else { @@ -425,7 +429,7 @@ impl D<'_> { self.rv(o.rm) }, ); - if op & 2 != 0 { + if op & 2 != 0 && !matches!(op, 0x86 | 0x87) { format!("{name} {reg}, {rm}") } else { format!("{name} {rm}, {reg}") @@ -815,6 +819,27 @@ mod tests { ); } #[test] + fn xchg_roundtrip_in_every_width() { + // Byte sequences checked against llvm-mc. + let mut e = Encoder::default(); + e.xchg(R11, 8, R10, 64); + assert_eq!(e.code, [0x4d, 0x87, 0x93, 8, 0, 0, 0]); + e.xchg(R11, 8, R10, 32); + e.xchg(R11, 8, R10, 16); + e.xchg(R11, 8, R10, 8); + e.xchg(RCX, 0, RAX, 8); + e.xchg(RCX, 0, RSI, 8); + assert_eq!( + format(&e.code), + "0000: xchg qword ptr [r11 + 8], r10\n\ + 0007: xchg dword ptr [r11 + 8], r10d\n\ + 000e: xchg word ptr [r11 + 8], r10w\n\ + 0016: xchg byte ptr [r11 + 8], r10b\n\ + 001d: xchg byte ptr [rcx], al\n\ + 0024: xchg byte ptr [rcx], sil\n" + ); + } + #[test] fn xmm_and_memory_roundtrip() { let mut e = Encoder::default(); e.xmm_mem(13, RBP, -32, 8, false); diff --git a/src/target/src/x86_64/encode.rs b/src/target/src/x86_64/encode.rs index f6983f3..4e22e6d 100644 --- a/src/target/src/x86_64/encode.rs +++ b/src/target/src/x86_64/encode.rs @@ -70,28 +70,13 @@ impl Encoder { self.code.push(0xb8 | (dst & 7)); self.bytes(&imm.to_le_bytes()); } - pub fn load(&mut self, dst: Reg, base: Reg, off: i32, bits: u16, signed: bool) { - match (bits, signed) { - (1..=8, s) => self.mem( - None, - &[0x0f, if s { 0xbe } else { 0xb6 }], - dst, - base, - off, - true, - false, - ), - (9..=16, s) => self.mem( - None, - &[0x0f, if s { 0xbf } else { 0xb7 }], - dst, - base, - off, - true, - false, - ), - (17..=32, true) => self.mem(None, &[0x63], dst, base, off, true, false), - (17..=32, false) => self.mem(None, &[0x8b], dst, base, off, false, false), + /// Loads `bits` of memory into `dst`, zero-extending to 64 bits. Sign extension is an + /// explicit operation of the instruction selector, never part of a load. + pub fn load(&mut self, dst: Reg, base: Reg, off: i32, bits: u16) { + match bits { + 1..=8 => self.mem(None, &[0x0f, 0xb6], dst, base, off, true, false), + 9..=16 => self.mem(None, &[0x0f, 0xb7], dst, base, off, true, false), + 17..=32 => self.mem(None, &[0x8b], dst, base, off, false, false), _ => self.mem(None, &[0x8b], dst, base, off, true, false), } } @@ -110,6 +95,23 @@ impl Encoder { bits <= 8, ); } + /// `xchg [base + off], src` of `bits` (8, 16, 32 or 64): swaps `src` with memory. With a + /// memory operand the exchange carries an implicit `lock`, so it is a full barrier. + pub fn xchg(&mut self, base: Reg, off: i32, src: Reg, bits: u16) { + self.mem( + if bits > 8 && bits <= 16 { + Some(0x66) + } else { + None + }, + &[if bits <= 8 { 0x86 } else { 0x87 }], + src, + base, + off, + bits > 32, + bits <= 8, + ); + } pub fn push(&mut self, reg: Reg) { self.rex(false, 0, reg, false); self.code.push(0x50 | (reg & 7)); diff --git a/src/target/src/x86_64/isel.rs b/src/target/src/x86_64/isel.rs index 575ea9f..5545fa4 100644 --- a/src/target/src/x86_64/isel.rs +++ b/src/target/src/x86_64/isel.rs @@ -10,6 +10,7 @@ use crate::regalloc::wimmer::{ UseKind, }; use alloc::{vec, vec::Vec}; +use volt_ir::attribute::Endianness; use volt_ir::function::*; use volt_ir::types::{FloatKind, Type, TypeKind}; @@ -88,7 +89,7 @@ fn shape(f: &Function, t: Type) -> Result<(usize, bool), Error> { && matches!( f.types.type_kind(v.elem), TypeKind::Float(FloatKind::F32) - | TypeKind::Int(volt_ir::types::IntDesc { bits: 32, .. }) + | TypeKind::Int(volt_ir::types::IntDesc { bits: 32 }) ) => { ((v.len * 4) as usize, true) @@ -114,6 +115,31 @@ fn size(f: &Function, t: Type) -> Result { _ => Ok(shape(f, t)?.0), } } +/// `n` truncated to `bits` and zero-extended: the canonical register form of an integer. +fn truncated(n: i64, bits: u16) -> u64 { + if bits < 64 { + n as u64 & ((1u64 << bits) - 1) + } else { + n as u64 + } +} +/// Condition code of an integer compare: equality, signed order (`l`, `le`, `g`, `ge`) or +/// unsigned order (`b`, `be`, `a`, `ae`). The floating-point orders have no integer meaning. +fn int_condition(op: CmpOp) -> Result { + Ok(match op { + CmpOp::Eq => 4, + CmpOp::Ne => 5, + CmpOp::Slt => 12, + CmpOp::Sle => 14, + CmpOp::Sgt => 15, + CmpOp::Sge => 13, + CmpOp::Ult => 2, + CmpOp::Ule => 6, + CmpOp::Ugt => 7, + CmpOp::Uge => 3, + CmpOp::Lt | CmpOp::Le | CmpOp::Gt | CmpOp::Ge => return Err(Error::InvalidFunction), + }) +} struct Ctx<'a> { f: &'a Function, e: Encoder, @@ -151,9 +177,6 @@ impl Ctx<'_> { _ => 64, } } - fn signed(&self, v: Value) -> bool { - matches!(self.ty(v),TypeKind::Int(i) if i.signed) - } fn wide(&self, v: Value) -> bool { self.bytes(v) == 32 } @@ -181,7 +204,7 @@ impl Ctx<'_> { Location::Reg(s) => self.e.mov(r, s as u8), Location::Slot(s) => { let d = self.slot(0, s); - self.e.load(r, RBP, d, 64, false); + self.e.load(r, RBP, d, 64); } } } @@ -240,7 +263,7 @@ impl Ctx<'_> { self.e .xmm_mem(d as u8, RBP, off, if wide { 32 } else { 16 }, false) } else { - self.e.load(d as u8, RBP, off, 64, false) + self.e.load(d as u8, RBP, off, 64) } } (Location::Reg(s), Location::Slot(d)) => { @@ -261,7 +284,7 @@ impl Ctx<'_> { self.e .xmm_mem(15, RBP, de, if wide { 32 } else { 16 }, true) } else { - self.e.load(R11, RBP, so, 64, false); + self.e.load(R11, RBP, so, 64); self.e.store(RBP, de, R11, 64) } } @@ -282,15 +305,28 @@ impl Ctx<'_> { let at = self.e.branch(None); self.fixups.push((at, j.target)); } + /// Extends the low `bits` of `r` to 64 bits, sign-extending when `signed` and + /// zero-extending otherwise. Every integer narrower than 64 bits is held in its register + /// zero-extended; an operation that must read an operand as signed extends a scratch copy + /// of it with this first (`R10`/`R11` always hold such copies). fn normalize(&mut self, r: u8, bits: u16, signed: bool) { - if bits < 64 { - let shift = (64 - bits) as u8; - self.e.shift(4, r, Some(shift), true); - self.e - .shift(if signed { 7 } else { 5 }, r, Some(shift), true); + match (bits, signed) { + (64.., _) => {} + (32, false) => self.e.rr(None, &[0x89], r, r, false), + (32, true) => self.e.rr(None, &[0x63], r, r, true), + _ => { + let shift = (64 - bits) as u8; + self.e.shift(4, r, Some(shift), true); + self.e + .shift(if signed { 7 } else { 5 }, r, Some(shift), true); + } } } - fn int_binary_raw(&mut self, op: BinOp, wide: bool, signed: bool) { + /// `r10 = r10 r11` in 64-bit (`wide`) or 32-bit registers. A signed divide, remainder + /// or arithmetic shift reads its 32/64-bit operands as two's complement, which a + /// zero-extended 32-bit value already is; narrower operands must be sign-extended by the + /// caller (see `int_binary`). + fn int_binary_raw(&mut self, op: BinOp, wide: bool) -> Result<(), Error> { match op { BinOp::Add | BinOp::Sub | BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => self.e.rr( None, @@ -306,52 +342,66 @@ impl Ctx<'_> { wide, ), BinOp::Mul => self.e.rr(None, &[0x0f, 0xaf], R10, R11, wide), - BinOp::Shl | BinOp::Shr => { + BinOp::Shl | BinOp::Shr | BinOp::Sar => { self.e.mov(RCX, R11); self.e.shift( - if op == BinOp::Shl { - 4 - } else if signed { - 7 - } else { - 5 + match op { + BinOp::Shl => 4, + BinOp::Shr => 5, + _ => 7, }, R10, None, wide, ); } - BinOp::Div | BinOp::Rem => { + BinOp::UDiv | BinOp::URem => { self.e.mov(RAX, R10); - if signed { - if wide { - self.e.bytes(&[0x48, 0x99]); - } else { - self.e.bytes(&[0x99]); - } + self.e.rr(None, &[0x31], RDX, RDX, false); + self.e.rr(None, &[0xf7], 6, R11, wide); + self.e.mov(R10, if op == BinOp::UDiv { RAX } else { RDX }); + } + BinOp::SDiv | BinOp::SRem => { + self.e.mov(RAX, R10); + if wide { + self.e.bytes(&[0x48, 0x99]); } else { - self.e.rr(None, &[0x31], RDX, RDX, false); + self.e.bytes(&[0x99]); } - self.e - .rr(None, &[0xf7], if signed { 7 } else { 6 }, R11, wide); - self.e.mov(R10, if op == BinOp::Div { RAX } else { RDX }); + self.e.rr(None, &[0xf7], 7, R11, wide); + self.e.mov(R10, if op == BinOp::SDiv { RAX } else { RDX }); } - BinOp::Mulh => { + BinOp::UMulh | BinOp::SMulh => { self.e.mov(RAX, R10); - self.e - .rr(None, &[0xf7], if signed { 5 } else { 4 }, R11, wide); + self.e.rr( + None, + &[0xf7], + if op == BinOp::SMulh { 5 } else { 4 }, + R11, + wide, + ); self.e.mov(R10, RDX); } + // Floating-point only: `verify` rejects them on integers. + BinOp::Div | BinOp::Rem => return Err(Error::InvalidFunction), } + Ok(()) } - fn int_binary(&mut self, op: BinOp, lhs: Value, result: Value) -> Result<(), Error> { + fn int_binary(&mut self, op: BinOp, result: Value) -> Result<(), Error> { let bits = self.bits(result); - let signed = self.signed(lhs); - if bits != 32 && bits != 64 { - self.normalize(R10, bits, signed); - self.normalize(R11, bits, signed); + let narrow = bits != 32 && bits != 64; + if narrow { + match op { + BinOp::SDiv | BinOp::SRem | BinOp::SMulh => { + self.normalize(R10, bits, true); + self.normalize(R11, bits, true); + } + BinOp::Sar => self.normalize(R10, bits, true), + _ => {} + } } - if op == BinOp::Mulh && bits != 32 && bits != 64 { + if narrow && matches!(op, BinOp::UMulh | BinOp::SMulh) { + let signed = op == BinOp::SMulh; if bits < 32 { self.e.rr(None, &[0x0f, 0xaf], R10, R11, true); self.e @@ -366,38 +416,44 @@ impl Ctx<'_> { self.e.rr(None, &[0x09], RDX, R10, true); } } else { - self.int_binary_raw(op, bits > 32, signed); + self.int_binary_raw(op, bits > 32)?; } - if bits != 32 && bits != 64 { - self.normalize(R10, bits, self.signed(result)); + // Only these can leave bits above `bits` set in the result. + if narrow + && matches!( + op, + BinOp::Add + | BinOp::Sub + | BinOp::Mul + | BinOp::Shl + | BinOp::SDiv + | BinOp::SRem + | BinOp::Sar + | BinOp::SMulh + ) + { + self.normalize(R10, bits, false); } self.put(result, R10); Ok(()) } - fn integer_vector(&self, v: Value) -> Option { - match self.ty(v) { - TypeKind::Vector(v) => match self.f.types.type_kind(v.elem) { - TypeKind::Int(i) => Some(i.signed), - _ => None, - }, - _ => None, - } + fn integer_vector(&self, v: Value) -> bool { + matches!(self.ty(v), TypeKind::Vector(v) + if matches!(self.f.types.type_kind(v.elem), TypeKind::Int(_))) } - fn vector_integer_binary( - &mut self, - op: BinOp, - result: Value, - signed: bool, - ) -> Result<(), Error> { + fn scalar_float(&self, v: Value) -> bool { + matches!(self.ty(v), TypeKind::Float(k) if !matches!(k, FloatKind::F128)) + } + fn vector_integer_binary(&mut self, op: BinOp, result: Value) -> Result<(), Error> { let left = self.disp(self.temp); let right = self.disp(self.temp + 32); let out = self.disp(self.temp + 64); self.e.xmm_mem(13, RBP, left, self.bytes(result), true); self.e.xmm_mem(14, RBP, right, self.bytes(result), true); for lane in 0..self.bytes(result) / 4 { - self.e.load(R10, RBP, left + lane as i32 * 4, 32, signed); - self.e.load(R11, RBP, right + lane as i32 * 4, 32, signed); - self.int_binary_raw(op, false, signed); + self.e.load(R10, RBP, left + lane as i32 * 4, 32); + self.e.load(R11, RBP, right + lane as i32 * 4, 32); + self.int_binary_raw(op, false)?; self.e.store(RBP, out + lane as i32 * 4, R10, 32); } self.e.xmm_mem(13, RBP, out, self.bytes(result), false); @@ -405,8 +461,8 @@ impl Ctx<'_> { Ok(()) } fn float_binary(&mut self, op: BinOp, result: Value) -> Result<(), Error> { - if let Some(signed) = self.integer_vector(result) { - return self.vector_integer_binary(op, result, signed); + if self.integer_vector(result) { + return self.vector_integer_binary(op, result); } let opcode = match op { BinOp::Add => 0x58, @@ -439,10 +495,39 @@ impl Ctx<'_> { self.xput(result, 13); Ok(()) } + /// Rejects an ordered access with no instruction sequence: an ordering the access kind + /// cannot carry, or a type other than a 1/2/4/8-byte integer or pointer (legalize has + /// already turned atomic floats into integer accesses). + fn check_ordering(&self, mem: &MemFlags, v: Value, store: bool) -> Result<(), Error> { + let Some(ordering) = mem.ordering else { + return Ok(()); + }; + let scalar = match self.ty(v) { + TypeKind::Int(i) => matches!(i.bits, 8 | 16 | 32 | 64), + TypeKind::Ptr(_) => true, + _ => false, + }; + let allowed = if store { + matches!( + ordering, + AtomicOrdering::Relaxed | AtomicOrdering::Release | AtomicOrdering::SeqCst + ) + } else { + matches!( + ordering, + AtomicOrdering::Relaxed | AtomicOrdering::Acquire | AtomicOrdering::SeqCst + ) + }; + if scalar && allowed { + Ok(()) + } else { + Err(Error::Unsupported("unorderable atomic access")) + } + } fn memory_load(&mut self, result: Value, base: u8, off: i32) { if self.fp(result) { if self.half(result) { - self.e.load(R10, base, off, 16, false); + self.e.load(R10, base, off, 16); self.e.rr(Some(0x66), &[0x0f, 0x6e], 13, R10, false); self.e.vex_rr(2, 1, 0x13, 13, 0, 13, false); } else { @@ -450,8 +535,12 @@ impl Ctx<'_> { } self.xput(result, 13); } else { - self.e - .load(R10, base, off, self.bits(result), self.signed(result)); + let bits = self.bits(result); + self.e.load(R10, base, off, bits); + // A load reads a whole byte, word or doubleword: clear what lies above an odd width. + if !matches!(bits, 8 | 16 | 32 | 64) { + self.normalize(R10, bits, false); + } self.put(result, R10); } } @@ -471,7 +560,8 @@ impl Ctx<'_> { self.e.store(base, off, R10, self.bits(value)); } } - fn integer_vector_compare(&mut self, c: Compare, r: Value, signed: bool) { + fn integer_vector_compare(&mut self, c: Compare, r: Value) -> Result<(), Error> { + let cc = int_condition(c.op)?; let left = self.disp(self.temp); let right = self.disp(self.temp + 32); let out = self.disp(self.temp + 64); @@ -479,41 +569,9 @@ impl Ctx<'_> { self.xget(c.rhs, 14); self.e.xmm_mem(13, RBP, left, self.bytes(c.lhs), true); self.e.xmm_mem(14, RBP, right, self.bytes(c.rhs), true); - let cc = match c.op { - CmpOp::Eq => 4, - CmpOp::Ne => 5, - CmpOp::Lt => { - if signed { - 12 - } else { - 2 - } - } - CmpOp::Le => { - if signed { - 14 - } else { - 6 - } - } - CmpOp::Gt => { - if signed { - 15 - } else { - 7 - } - } - CmpOp::Ge => { - if signed { - 13 - } else { - 3 - } - } - }; for lane in 0..self.bytes(c.lhs) / 4 { - self.e.load(R10, RBP, left + lane as i32 * 4, 32, false); - self.e.load(R11, RBP, right + lane as i32 * 4, 32, false); + self.e.load(R10, RBP, left + lane as i32 * 4, 32); + self.e.load(R11, RBP, right + lane as i32 * 4, 32); self.e.rr(None, &[0x39], R11, R10, false); self.e.setcc(cc, R10); self.e.rr(None, &[0xf7], 3, R10, false); @@ -521,16 +579,23 @@ impl Ctx<'_> { } self.e.xmm_mem(13, RBP, out, self.bytes(r), false); self.xput(r, 13); + Ok(()) } fn comparison(&mut self, c: Compare, r: Value) -> Result<(), Error> { - if let Some(signed) = self.integer_vector(c.lhs) { - self.integer_vector_compare(c, r, signed); - return Ok(()); + if self.integer_vector(c.lhs) { + return self.integer_vector_compare(c, r); } if self.fp(c.lhs) { self.xget(c.lhs, 13); self.xget(c.rhs, 14); if matches!(self.ty(c.lhs), TypeKind::Vector(_)) { + let predicate = match c.op { + CmpOp::Eq => 0, + CmpOp::Ne => 4, + CmpOp::Lt | CmpOp::Gt => 1, + CmpOp::Le | CmpOp::Ge => 2, + _ => return Err(Error::InvalidFunction), + }; let swap = matches!(c.op, CmpOp::Gt | CmpOp::Ge); let (l, h) = if swap { (14, 13) } else { (13, 14) }; if self.wide(c.lhs) { @@ -539,12 +604,7 @@ impl Ctx<'_> { self.e.xmm_mov(15, l, false); self.e.rr(None, &[0x0f, 0xc2], 15, h, false); } - self.e.code.push(match c.op { - CmpOp::Eq => 0, - CmpOp::Ne => 4, - CmpOp::Lt | CmpOp::Gt => 1, - _ => 2, - }); + self.e.code.push(predicate); self.xput(r, 15); return Ok(()); } @@ -553,13 +613,6 @@ impl Ctx<'_> { "f128 compare requires softfp legalization", )); } - self.e.rr( - if self.double(c.lhs) { Some(0x66) } else { None }, - &[0x0f, 0x2e], - 13, - 14, - false, - ); let cc = match c.op { CmpOp::Eq => 4, CmpOp::Ne => 5, @@ -567,7 +620,15 @@ impl Ctx<'_> { CmpOp::Le => 6, CmpOp::Gt => 7, CmpOp::Ge => 3, + _ => return Err(Error::InvalidFunction), }; + self.e.rr( + if self.double(c.lhs) { Some(0x66) } else { None }, + &[0x0f, 0x2e], + 13, + 14, + false, + ); self.e.setcc(cc, R10); if matches!(c.op, CmpOp::Eq | CmpOp::Lt | CmpOp::Le) { self.e.setcc(11, R11); @@ -578,63 +639,47 @@ impl Ctx<'_> { } self.put(r, R10); } else { + let cc = int_condition(c.op)?; + let bits = self.bits(c.lhs); self.get(c.lhs, R10); self.get(c.rhs, R11); - self.e.rr(None, &[0x39], R11, R10, self.bits(c.lhs) > 32); - let signed = self.signed(c.lhs); - let cc = match c.op { - CmpOp::Eq => 4, - CmpOp::Ne => 5, - CmpOp::Lt => { - if signed { - 12 - } else { - 2 - } - } - CmpOp::Le => { - if signed { - 14 - } else { - 6 - } - } - CmpOp::Gt => { - if signed { - 15 - } else { - 7 - } - } - CmpOp::Ge => { - if signed { - 13 - } else { - 3 - } - } - }; + // A signed order reads the operands as two's complement of `bits`, which the + // zero-extended form only is at 32 and 64 bits. + if matches!(c.op, CmpOp::Slt | CmpOp::Sle | CmpOp::Sgt | CmpOp::Sge) + && bits != 32 + && bits != 64 + { + self.normalize(R10, bits, true); + self.normalize(R11, bits, true); + } + self.e.rr(None, &[0x39], R11, R10, bits > 32); self.e.setcc(cc, R10); self.put(r, R10); } Ok(()) } - fn convert(&mut self, s: Value, r: Value) -> Result<(), Error> { - match (self.fp(s), self.fp(r)) { - (false, false) => { + fn convert(&mut self, c: Convert, r: Value) -> Result<(), Error> { + let s = c.value; + let (from, to) = (self.bits(s), self.bits(r)); + match c.kind { + ConvertKind::Trunc | ConvertKind::Zext | ConvertKind::Sext => { + if self.fp(s) || self.fp(r) { + return Err(Error::InvalidFunction); + } self.get(s, R10); - if self.bits(r) > self.bits(s) { - self.normalize(R10, self.bits(s), self.signed(s)); + match c.kind { + // The source is already zero-extended: nothing to do. + ConvertKind::Zext => {} + ConvertKind::Trunc => self.normalize(R10, to, false), + _ => { + self.normalize(R10, from, true); + self.normalize(R10, to, false); + } } - self.normalize(R10, self.bits(r), self.signed(r)); self.put(r, R10); } - (true, true) => { - if matches!(self.ty(s), TypeKind::Vector(_)) - || matches!(self.ty(r), TypeKind::Vector(_)) - || self.bytes(s) == 16 - || self.bytes(r) == 16 - { + ConvertKind::FpResize => { + if !self.scalar_float(s) || !self.scalar_float(r) { return Err(Error::Unsupported("composite conversion")); } self.xget(s, 13); @@ -652,11 +697,18 @@ impl Ctx<'_> { } self.xput(r, 13); } - (false, true) => { + ConvertKind::SiToFp | ConvertKind::UiToFp => { + if self.fp(s) || !self.scalar_float(r) { + return Err(Error::Unsupported("composite conversion")); + } self.get(s, R10); - self.normalize(R10, self.bits(s), self.signed(s)); let p = Some(if self.double(r) { 0xf2 } else { 0xf3 }); - if !self.signed(s) && self.bits(s) == 64 { + if c.kind == ConvertKind::SiToFp { + self.normalize(R10, from, true); + self.e.rr(p, &[0x0f, 0x2a], 13, R10, true); + } else if from == 64 { + // A value with the top bit set does not fit a signed convert: halve it + // (keeping the low bit for rounding), convert, and double the result. self.e.rr(None, &[0x85], R10, R10, true); let normal = self.e.branch(Some(9)); self.e.mov(R11, R10); @@ -677,10 +729,14 @@ impl Ctx<'_> { } self.xput(r, 13); } - (true, false) => { + ConvertKind::FpToSi | ConvertKind::FpToUi => { + if !self.scalar_float(s) || self.fp(r) { + return Err(Error::Unsupported("composite conversion")); + } self.xget(s, 13); let prefix = Some(if self.double(s) { 0xf2 } else { 0xf3 }); - if !self.signed(r) && self.bits(r) == 64 { + let unsigned = c.kind == ConvertKind::FpToUi; + if unsigned && to == 64 { self.e.imm( R10, if self.double(s) { @@ -708,15 +764,10 @@ impl Ctx<'_> { self.e.rr(prefix, &[0x0f, 0x2c], R10, 13, true); self.e.patch(end, self.e.code.len())?; } else { - self.e.rr( - prefix, - &[0x0f, 0x2c], - R10, - 13, - self.bits(r) > 32 || !self.signed(r), - ); + self.e + .rr(prefix, &[0x0f, 0x2c], R10, 13, to > 32 || unsigned); } - self.normalize(R10, self.bits(r), false); + self.normalize(R10, to, false); self.put(r, R10); } } @@ -747,7 +798,7 @@ impl Ctx<'_> { if self.fp(v) { if xi < 8 { if self.half(v) { - self.e.load(R10, RBP, off, 16, false); + self.e.load(R10, RBP, off, 16); self.e.rr(Some(0x66), &[0x0f, 0x6e], xi, R10, false); } else { self.e.xmm_mem(xi, RBP, off, self.bytes(v), false); @@ -762,16 +813,11 @@ impl Ctx<'_> { stack += side; } } else if gi < 6 { - self.e.load( - [RDI, RSI, RDX, RCX, R8, R9][gi], - RBP, - off, - self.bits(v), - self.signed(v), - ); + self.e + .load([RDI, RSI, RDX, RCX, R8, R9][gi], RBP, off, self.bits(v)); gi += 1; } else { - self.e.load(R11, RBP, off, 64, false); + self.e.load(R11, RBP, off, 64); self.e.store(RSP, stack, R11, 64); stack += 8; } @@ -780,7 +826,7 @@ impl Ctx<'_> { self.e.imm(RAX, xi as u64); } if target.is_some() { - self.e.load(R10, RBP, self.disp(self.temp), 64, false); + self.e.load(R10, RBP, self.disp(self.temp), 64); self.e.rr(None, &[0xff], 2, R10, false); } else { self.e.code.push(0xe8); @@ -803,6 +849,8 @@ impl Ctx<'_> { } self.xput(r, 0); } else { + // The ABI leaves the bits above a narrow integer return undefined. + self.normalize(RAX, self.bits(r), false); self.put(r, RAX); } } @@ -884,18 +932,18 @@ impl Ctx<'_> { let limit = if fp { 176 } else { 48 }; let step = if fp { 16 } else { 8 }; self.get(list, R11); - self.e.load(R10, R11, field, 32, false); + self.e.load(R10, R11, field, 32); self.e.alu_imm(7, R10, limit, false); let overflow = self.e.branch(Some(3)); self.e.mov(RCX, R10); - self.e.load(RAX, R11, 16, 64, false); + self.e.load(RAX, R11, 16, 64); self.e.rr(None, &[0x01], R10, RAX, true); self.e.alu_imm(0, RCX, step, false); self.e.store(R11, field, RCX, 32); self.memory_load(result, RAX, 0); let end = self.e.branch(None); self.e.patch(overflow, self.e.code.len())?; - self.e.load(RAX, R11, 8, 64, false); + self.e.load(RAX, R11, 8, 64); self.e.mov(RCX, RAX); self.e.alu_imm(0, RCX, 8, true); self.e.store(R11, 8, RCX, 64); @@ -921,8 +969,22 @@ impl Ctx<'_> { let op = self.f.opcode(i); match op { Opcode::Store(s) => { + if s.mem.endian == Endianness::Big { + return Err(Error::Unsupported("big-endian store")); + } + self.check_ordering(&s.mem, s.value, true)?; self.get(s.ptr, R11); - self.memory_store(s.value, R11, self.fold.off_of(i) as i32); + let off = self.fold.off_of(i) as i32; + if s.mem.ordering == Some(AtomicOrdering::SeqCst) { + // x86-64 (TSO) already orders relaxed and release stores with plain `mov`; + // only a sequentially consistent store must also not reorder with a later + // load, which `xchg` (implicitly locked) guarantees. It leaves the old + // memory value in R10, a scratch register. + self.get(s.value, R10); + self.e.xchg(R11, off, R10, self.bits(s.value)); + } else { + self.memory_store(s.value, R11, off); + } } Opcode::Prefetch(p) => { self.get(p.ptr, R11); @@ -938,12 +1000,12 @@ impl Ctx<'_> { } Opcode::VaStart(v) => self.va_start(v.list), Opcode::VaEnd(_) => {} + Opcode::Intrinsic(_) => return Err(Error::Unsupported("intrinsic")), _ => { let r = result.ok_or(Error::Unsupported("statement"))?; match op { Opcode::Iconst(n) => { - self.e.imm(R10, n as u64); - self.normalize(R10, self.bits(r), self.signed(r)); + self.e.imm(R10, truncated(n, self.bits(r))); self.put(r, R10); } Opcode::Fconst(n) => { @@ -980,7 +1042,7 @@ impl Ctx<'_> { } else { self.get(a.lhs, R10); self.get(a.rhs, R11); - self.int_binary(a.op, a.lhs, r)?; + self.int_binary(a.op, r)?; } } Opcode::ArithImm(a) => { @@ -988,8 +1050,8 @@ impl Ctx<'_> { return Err(Error::Unsupported("floating immediate arithmetic")); } self.get(a.lhs, R10); - self.e.imm(R11, a.imm as u64); - self.int_binary(a.op, a.lhs, r)?; + self.e.imm(R11, truncated(a.imm, self.bits(r))); + self.int_binary(a.op, r)?; } Opcode::Icmp(c) => self.comparison(c, r)?, Opcode::Select(s) => { @@ -1018,7 +1080,7 @@ impl Ctx<'_> { self.e.patch(end, self.e.code.len())?; } } - Opcode::Convert(c) => self.convert(c.value, r)?, + Opcode::Convert(c) => self.convert(c, r)?, Opcode::Unary(u) => { if u.op == UnaryOp::Reinterpret { if self.fp(u.value) == self.fp(r) { @@ -1088,6 +1150,13 @@ impl Ctx<'_> { } } Opcode::Load(l) => { + if l.mem.endian == Endianness::Big { + return Err(Error::Unsupported("big-endian load")); + } + self.check_ordering(&l.mem, r, false)?; + // Every permitted load ordering is a plain `mov` on x86-64: TSO gives + // loads acquire semantics, and a sequentially consistent load needs + // no extra instruction because the store side is the locked one. self.get(l.ptr, R11); self.memory_load(r, R11, self.fold.off_of(i) as i32); } @@ -1158,15 +1227,18 @@ impl Ctx<'_> { } self.xput(r, 13); } else { - self.e.load(R10, RBP, off, 32, false); + self.e.load(R10, RBP, off, 32); for j in 1..self.bytes(rd.vector) / 4 { - self.e.load(R11, RBP, off + (j * 4) as i32, 32, false); - self.int_binary(rd.op, r, r)?; + self.e.load(R11, RBP, off + (j * 4) as i32, 32); + self.int_binary(rd.op, r)?; } self.put(r, R10); } } Opcode::Dot(d) => { + if self.integer_vector(d.a) { + return Err(Error::Unsupported("integer dot product")); + } let off = self.disp(self.temp); self.xget(d.a, 13); self.xget(d.b, 14); @@ -1209,7 +1281,7 @@ impl Ctx<'_> { return Err(Error::Unsupported("too many FP returns")); } if self.half(v) { - self.e.load(R10, RBP, off, 16, false); + self.e.load(R10, RBP, off, 16); self.e.rr(Some(0x66), &[0x0f, 0x6e], xi, R10, false); } else { self.e.xmm_mem(xi, RBP, off, self.bytes(v), false); @@ -1219,13 +1291,12 @@ impl Ctx<'_> { if gi >= 2 { return Err(Error::Unsupported("too many integer returns")); } - self.e - .load([RAX, RDX][gi], RBP, off, self.bits(v), self.signed(v)); + self.e.load([RAX, RDX][gi], RBP, off, self.bits(v)); gi += 1; } } if let Some(home) = self.sret_home { - self.e.load(RAX, RBP, self.disp(home), 64, false); + self.e.load(RAX, RBP, self.disp(home), 64); } if self.stack_alignment > 16 { self.e.mem( @@ -1492,7 +1563,7 @@ fn compile_lowered( c.e.store(RBP, off, [RDI, RSI, RDX, RCX, R8, R9][gi], 64); gi += 1; } else { - c.e.load(R10, RBP, stack, 64, false); + c.e.load(R10, RBP, stack, 64); c.e.store(RBP, off, R10, 64); stack += 8; } @@ -1542,3 +1613,131 @@ fn compile_lowered( relocs: c.relocs, }) } + +/// Byte-level checks of ordered accesses. They only compile code, so unlike `tests.rs` (which +/// executes it) they run on every host. +#[cfg(test)] +mod ordered_access_codegen { + use crate::x86_64::{Error, compile}; + use volt_ir::function::*; + + fn store_fn(kind: &str, ordering: Option) -> Function { + let mut f = Function::new(); + let p = f.types.parse_type("ptr").unwrap(); + let t = f.types.parse_type(kind).unwrap(); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let x = f.append_block_param(b, t); + let mem = MemFlags { + ordering, + ..MemFlags::new() + }; + f.append_store_mem(b, x, ptr, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + f + } + + fn load_fn(kind: &str, ordering: Option) -> Function { + let mut f = Function::new(); + let p = f.types.parse_type("ptr").unwrap(); + let t = f.types.parse_type(kind).unwrap(); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let mem = MemFlags { + ordering, + ..MemFlags::new() + }; + let v = f.append_load(b, t, ptr, mem); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + f + } + + fn contains(code: &[u8], pattern: &[u8]) -> bool { + code.windows(pattern.len()).any(|w| w == pattern) + } + + /// `[prefix] REX opcode modrm disp32=0` for `[r11]` and `r10`: the shape of both the plain + /// `mov [r11], r10` store and the `xchg [r11], r10` the encoder emits (R10 holds the value, + /// R11 the address). + fn r11_r10(prefix: &[u8], rex: u8, opcode: u8) -> Vec { + let mut bytes = prefix.to_vec(); + bytes.extend_from_slice(&[rex, opcode, 0x93, 0, 0, 0, 0]); + bytes + } + + #[test] + fn seq_cst_stores_are_xchg_and_weaker_stores_are_plain_mov() { + // (type, operand-size prefix, REX, xchg opcode, mov opcode) + for (kind, prefix, rex, xchg, mov) in [ + ("i8", &[][..], 0x45u8, 0x86u8, 0x88u8), + ("i16", &[0x66][..], 0x45, 0x87, 0x89), + ("i32", &[][..], 0x45, 0x87, 0x89), + ("i64", &[][..], 0x4d, 0x87, 0x89), + ] { + let xchg_bytes = r11_r10(prefix, rex, xchg); + let mov_bytes = r11_r10(prefix, rex, mov); + let plain = compile(&store_fn(kind, None)).unwrap(); + assert!(contains(&plain, &mov_bytes), "{kind} plain store"); + assert!(!contains(&plain, &xchg_bytes), "{kind} plain store"); + for ordering in [AtomicOrdering::Relaxed, AtomicOrdering::Release] { + let code = compile(&store_fn(kind, Some(ordering))).unwrap(); + assert_eq!(code, plain, "{kind} {ordering:?} must be a plain store"); + } + let seq_cst = compile(&store_fn(kind, Some(AtomicOrdering::SeqCst))).unwrap(); + assert!(contains(&seq_cst, &xchg_bytes), "{kind} seq_cst store"); + assert!(!contains(&seq_cst, &mov_bytes), "{kind} seq_cst store"); + assert_eq!(seq_cst.len(), plain.len(), "{kind}: xchg replaces the mov"); + } + } + + #[test] + fn ordered_loads_are_plain_movs() { + for kind in ["i8", "i16", "i32", "i64", "ptr"] { + let plain = compile(&load_fn(kind, None)).unwrap(); + for ordering in [ + AtomicOrdering::Relaxed, + AtomicOrdering::Acquire, + AtomicOrdering::SeqCst, + ] { + let code = compile(&load_fn(kind, Some(ordering))).unwrap(); + assert_eq!(code, plain, "{kind} {ordering:?} must be a plain load"); + } + } + } + + #[test] + fn atomic_float_accesses_use_the_integer_register_path() { + // The float is reinterpreted into an integer register first, so an atomic f64 store + // compiles to the same `xchg` an atomic u64 store does. + let mut f = Function::new(); + let p = f.types.parse_type("ptr").unwrap(); + let t = f.types.parse_type("f64").unwrap(); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let x = f.append_block_param(b, t); + let mem = MemFlags { + ordering: Some(AtomicOrdering::SeqCst), + ..MemFlags::new() + }; + f.append_store_mem(b, x, ptr, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let code = compile(&f).unwrap(); + assert!(contains(&code, &r11_r10(&[], 0x4d, 0x87))); + } + + #[test] + fn orderings_without_an_instruction_fail_closed() { + for ordering in [AtomicOrdering::Release, AtomicOrdering::AcqRel] { + assert!(matches!( + compile(&load_fn("i32", Some(ordering))), + Err(Error::Unsupported(_)) + )); + } + for ordering in [AtomicOrdering::Acquire, AtomicOrdering::AcqRel] { + assert!(matches!( + compile(&store_fn("i32", Some(ordering))), + Err(Error::Unsupported(_)) + )); + } + } +} diff --git a/src/target/src/x86_64/link.rs b/src/target/src/x86_64/link.rs index 9756280..7886f66 100644 --- a/src/target/src/x86_64/link.rs +++ b/src/target/src/x86_64/link.rs @@ -6,74 +6,7 @@ use super::{ isel::{self, RelocKind}, }; use alloc::{string::String, vec::Vec}; -use volt_ir::function::Function; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DataKind { - Rodata, - Data, - Bss, -} -#[derive(Clone, Debug)] -pub struct DataReloc { - pub off: usize, - pub symbol: String, -} -#[derive(Clone, Debug)] -pub struct Data { - pub name: String, - pub bytes: Vec, - pub kind: DataKind, - pub size: u64, - pub relocs: Vec, -} -#[derive(Clone, Debug, Default)] -pub struct Module { - pub functions: Vec<(String, Function)>, - pub data: Vec, -} -impl Module { - pub fn new() -> Self { - Self::default() - } - pub fn add_function(&mut self, name: &str, function: Function) { - self.functions.push((name.into(), function)); - } - pub fn add_data(&mut self, name: &str, bytes: Vec) { - self.add_data_relocs(name, bytes, Vec::new()); - } - pub fn add_data_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.into(), - bytes, - kind: DataKind::Rodata, - size, - relocs, - }); - } - pub fn add_writable(&mut self, name: &str, bytes: Vec) { - self.add_writable_relocs(name, bytes, Vec::new()); - } - pub fn add_writable_relocs(&mut self, name: &str, bytes: Vec, relocs: Vec) { - let size = bytes.len() as u64; - self.data.push(Data { - name: name.into(), - bytes, - kind: DataKind::Data, - size, - relocs, - }); - } - pub fn add_bss(&mut self, name: &str, size: u64) { - self.data.push(Data { - name: name.into(), - bytes: Vec::new(), - kind: DataKind::Bss, - size, - relocs: Vec::new(), - }); - } -} +use volt_ir::module::{Global, Module}; #[derive(Clone, Debug)] pub struct Symbol { pub name: String, @@ -91,12 +24,13 @@ pub struct Reloc { pub struct Linked { pub code: Vec, pub symbols: Vec, - pub data: Vec, + pub data: Vec, pub relocs: Vec, } pub fn compile_module(module: &Module) -> Result { let mut result = Linked::default(); - for (name, f) in &module.functions { + for def in &module.functions { + let (name, f) = (&def.name, &def.function); while result.code.len() % 16 != 0 { result.code.push(0x90); } @@ -134,18 +68,34 @@ pub fn compile_module(module: &Module) -> Result { pending.push(r); } result.relocs = pending; - result.data = module.data.clone(); + result.data = module.globals.clone(); Ok(result) } #[cfg(test)] mod tests { use super::*; - use volt_ir::function::{Call, Opcode, Ret, RetPiece, Terminator}; + use volt_ir::function::{Call, Function, Opcode, Ret, RetPiece, Terminator}; + use volt_ir::module::DataKind; + #[test] + fn carries_globals_into_the_linked_image() { + let mut f = Function::new(); + let ty = f.types.parse_type("i64").unwrap(); + let block = f.append_block(); + let v = f.append_inst(block, ty, Opcode::Iconst(1)); + f.set_terminator(block, Terminator::Ret(Ret::one(v))); + let mut module = Module::new(); + module.add_function("f", f); + module.add_data("ro", alloc::vec![1, 2, 3]); + module.add_bss("zero", 32); + let linked = compile_module(&module).unwrap(); + assert_eq!(linked.data, module.globals); + assert_eq!(linked.data[1].kind, DataKind::Bss); + } #[test] fn resolves_inter_function_call_displacement() { let mut caller = Function::new(); - let ty = caller.types.parse_type("u64").unwrap(); + let ty = caller.types.parse_type("i64").unwrap(); let block = caller.append_block(); let symbol = caller.intern_symbol("callee"); let args = caller.intern_values(&[]); @@ -165,7 +115,7 @@ mod tests { ); caller.set_terminator(block, Terminator::Ret(Ret::one(result))); let mut callee = Function::new(); - let ty = callee.types.parse_type("u64").unwrap(); + let ty = callee.types.parse_type("i64").unwrap(); let block = callee.append_block(); let result = callee.append_inst(block, ty, Opcode::Iconst(42)); callee.set_terminator(block, Terminator::Ret(Ret::one(result))); diff --git a/src/target/src/x86_64/mod.rs b/src/target/src/x86_64/mod.rs index da9d20d..c98c4e5 100644 --- a/src/target/src/x86_64/mod.rs +++ b/src/target/src/x86_64/mod.rs @@ -36,5 +36,9 @@ pub fn compile(function: &volt_ir::function::Function) -> Result Result, Error> { let mut obj = Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); let mut symbols = Vec::new(); let mut pending = Vec::new(); - for (name, f) in &module.functions { + for def in &module.functions { + let (name, f) = (&def.name, &def.function); let compiled = isel::compile_object(f)?; let section = obj.add_section( Vec::new(), @@ -58,7 +59,7 @@ pub fn write_module(module: &Module) -> Result, Error> { )); } } - for data in &module.data { + for data in &module.globals { let (class, kind) = match data.kind { DataKind::Rodata => ("rodata", SectionKind::ReadOnlyData), DataKind::Data => ("data", SectionKind::Data), @@ -69,10 +70,11 @@ pub fn write_module(module: &Module) -> Result, Error> { format!(".{class}.{}", data.name.trim_start_matches('.')).into_bytes(), kind, ); + let align = data.effective_align().max(8) as u64; if data.kind == DataKind::Bss { - obj.append_section_bss(sec, data.size, 8); + obj.append_section_bss(sec, data.size, align); } else { - obj.append_section_data(sec, &data.bytes, 8); + obj.append_section_data(sec, &data.bytes, align); } let id = obj.add_symbol(Symbol { name: data.name.as_bytes().to_vec(), @@ -90,13 +92,13 @@ pub fn write_module(module: &Module) -> Result, Error> { }); symbols.push((data.name.clone(), id)); for r in &data.relocs { - if r.off + if r.offset .checked_add(8) .is_none_or(|end| end > data.size as usize) { return Err(Error::InvalidFunction); } - pending.push((sec, r.off as u64, r.symbol.clone(), R_X86_64_64, 0)); + pending.push((sec, r.offset as u64, r.symbol.clone(), R_X86_64_64, 0)); } } for (section, offset, name, r_type, addend) in pending { @@ -139,7 +141,7 @@ mod tests { #[test] fn elf_function_section_and_symbol() { let mut f = Function::new(); - let t = f.types.parse_type("u64").unwrap(); + let t = f.types.parse_type("i64").unwrap(); let b = f.append_block(); let v = f.append_inst(b, t, Opcode::Iconst(42)); f.set_terminator(b, Terminator::Ret(Ret::one(v))); @@ -210,4 +212,54 @@ mod tests { .any(|s| s.name().unwrap() == "external_function" && s.is_undefined()) ); } + #[test] + fn elf_emits_globals_with_data_relocations() { + use volt_ir::module::DataReloc; + let mut f = Function::new(); + let t = f.types.parse_type("i64").unwrap(); + let b = f.append_block(); + let v = f.append_inst(b, t, Opcode::Iconst(7)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let mut module = Module::new(); + module.add_function("seven", f); + module.add_data("ro", alloc::vec![1, 2, 3, 4]); + module.add_writable_relocs( + "table", + alloc::vec![0; 16], + alloc::vec![DataReloc { + offset: 8, + symbol: "seven".into(), + }], + ); + module.add_bss("scratch", 64); + let bytes = write_module(&module).unwrap(); + let elf = object::File::parse(bytes.as_slice()).unwrap(); + assert_eq!( + elf.section_by_name(".rodata.ro").unwrap().data().unwrap(), + &[1, 2, 3, 4] + ); + assert_eq!(elf.section_by_name(".bss.scratch").unwrap().size(), 64); + let table = elf.section_by_name(".data.table").unwrap(); + assert_eq!(table.size(), 16); + let relocations: Vec<_> = table.relocations().collect(); + assert_eq!(relocations.len(), 1); + assert_eq!(relocations[0].0, 8); + assert_eq!( + relocations[0].1.flags(), + RelocationFlags::Elf { + r_type: object::elf::RelocationType(R_X86_64_64) + } + ); + // A relocation slot that does not fit is rejected. + let mut bad = Module::new(); + bad.add_data_relocs( + "short", + alloc::vec![0; 4], + alloc::vec![DataReloc { + offset: 0, + symbol: "seven".into(), + }], + ); + assert!(matches!(write_module(&bad), Err(Error::InvalidFunction))); + } } diff --git a/src/target/src/x86_64/tests.rs b/src/target/src/x86_64/tests.rs index 95c5a81..18e6ee4 100644 --- a/src/target/src/x86_64/tests.rs +++ b/src/target/src/x86_64/tests.rs @@ -1,5 +1,6 @@ // Ported from Vulcan native/cases/link test boundaries (Apache-2.0). // Copyright LilithSemi; Rust implementation modified for Volt. +use super::cases::{self, Val}; use super::*; use volt_ir::function::*; use volt_ir::types::Type; @@ -45,39 +46,117 @@ impl Drop for Executable { fn ty(f: &mut Function, s: &str) -> Type { f.types.parse_type(s).unwrap() } -fn binary(kind: &str, op: BinOp) -> Function { - let mut f = Function::new(); - let t = ty(&mut f, kind); - let b = f.append_block(); - let l = f.append_block_param(b, t); - let r = f.append_block_param(b, t); - let out = f.append_inst(b, t, Opcode::Arith(Arith { op, lhs: l, rhs: r })); - f.set_terminator(b, Terminator::Ret(Ret::one(out))); - f +/// Calls `code` with `args`; the result's raw register contents (an `f32` in the low 32 bits). +fn call(code: &Executable, args: &[Val], want: Val) -> u64 { + use Val::*; + let p = code.ptr; + // SAFETY: every transmute names the System V signature of the function that was compiled + // for exactly this argument/result shape. + unsafe { + match (args, want) { + ([Int(a)], Int(_)) => { + let f: unsafe extern "C" fn(u64) -> u64 = core::mem::transmute(p); + f(*a) + } + ([Int(a), Int(b)], Int(_)) => { + let f: unsafe extern "C" fn(u64, u64) -> u64 = core::mem::transmute(p); + f(*a, *b) + } + ([Int(a), Int(b), Int(c), Int(d)], Int(_)) => { + let f: unsafe extern "C" fn(u64, u64, u64, u64) -> u64 = core::mem::transmute(p); + f(*a, *b, *c, *d) + } + ([F64(a)], Int(_)) => { + let f: unsafe extern "C" fn(f64) -> u64 = core::mem::transmute(p); + f(*a) + } + ([F32(a)], Int(_)) => { + let f: unsafe extern "C" fn(f32) -> u64 = core::mem::transmute(p); + f(*a) + } + ([Int(a)], F64(_)) => { + let f: unsafe extern "C" fn(u64) -> f64 = core::mem::transmute(p); + f(*a).to_bits() + } + ([Int(a)], F32(_)) => { + let f: unsafe extern "C" fn(u64) -> f32 = core::mem::transmute(p); + f(*a).to_bits() as u64 + } + ([F32(a)], F64(_)) => { + let f: unsafe extern "C" fn(f32) -> f64 = core::mem::transmute(p); + f(*a).to_bits() + } + ([F64(a)], F32(_)) => { + let f: unsafe extern "C" fn(f64) -> f32 = core::mem::transmute(p); + f(*a).to_bits() as u64 + } + _ => panic!("no calling convention for {args:?} -> {want:?}"), + } + } +} +/// Every operation whose result depends on signedness, at every width, against the reference +/// arithmetic of `cases`: arguments carry garbage above their declared width and results must +/// come back exactly zero-extended. +#[test] +fn signedness_cases_match_the_reference() { + for case in cases::all() { + let code = Executable::new(&case.function); + for (args, want) in &case.runs { + assert_eq!( + call(&code, args, *want), + want.bits(), + "{} {args:x?}", + case.name + ); + } + } } #[test] fn scalar_division_shifts_and_high_product() { for (kind, op, a, b, want) in [ - ("u64", BinOp::Div, u64::MAX, 2, u64::MAX / 2), - ("u64", BinOp::Rem, 100, 9, 1), - ("u64", BinOp::Shr, 1 << 63, 63, 1), - ("u64", BinOp::Shl, 7, 65, 14), - ("u64", BinOp::Mulh, u64::MAX, u64::MAX, u64::MAX - 1), - ("i64", BinOp::Div, (-100i64) as u64, 9, (-11i64) as u64), - ("i64", BinOp::Mulh, (-2i64) as u64, 3, u64::MAX), + ("i64", BinOp::UDiv, u64::MAX, 2, u64::MAX / 2), + ("i64", BinOp::URem, 100, 9, 1), + ("i64", BinOp::Shr, 1 << 63, 63, 1), + ("i64", BinOp::Sar, 1 << 63, 63, u64::MAX), + ("i64", BinOp::Shl, 7, 65, 14), + ("i64", BinOp::UMulh, u64::MAX, u64::MAX, u64::MAX - 1), + ("i64", BinOp::SDiv, (-100i64) as u64, 9, (-11i64) as u64), + ("i64", BinOp::SRem, (-100i64) as u64, 9, (-1i64) as u64), + ( + "i64", + BinOp::UDiv, + (-100i64) as u64, + 9, + ((-100i64) as u64) / 9, + ), + ("i64", BinOp::SMulh, (-2i64) as u64, 3, u64::MAX), ] { - let mem = Executable::new(&binary(kind, op)); + let mem = Executable::new(&cases::arith(kind, op)); // SAFETY: the compiled function has two 64-bit integer parameters and one result. let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { core::mem::transmute(mem.ptr) }; assert_eq!(unsafe { entry(a, b) }, want, "{kind} {op:?}"); } } +/// The one narrow signed divide that does not trap: the most negative value divided by -1 +/// wraps, because the 32-bit hardware divide it runs on has room for the true quotient. +#[test] +fn narrow_signed_divide_of_minimum_by_minus_one_wraps() { + for (kind, min) in [("i8", 0x80u64), ("i16", 0x8000)] { + let neg_one = u64::MAX; + for (op, want) in [(BinOp::SDiv, min), (BinOp::SRem, 0)] { + let mem = Executable::new(&cases::arith(kind, op)); + let entry: unsafe extern "C" fn(u64, u64) -> u64 = + unsafe { core::mem::transmute(mem.ptr) }; + assert_eq!(unsafe { entry(min, neg_one) }, want, "{kind} {op:?}"); + } + } +} #[test] fn narrow_store_keeps_neighbor_bytes() { let mut f = Function::new(); let p = ty(&mut f, "ptr"); - let byte = ty(&mut f, "u8"); - let word = ty(&mut f, "u32"); + let byte = ty(&mut f, "i8"); + let word = ty(&mut f, "i32"); let b = f.append_block(); let ptr = f.append_block_param(b, p); let v = f.append_inst(b, byte, Opcode::Iconst(0xab)); @@ -87,7 +166,7 @@ fn narrow_store_keeps_neighbor_bytes() { word, Opcode::Load(Load { ptr, - volatile: false, + mem: MemFlags::new(), }), ); f.set_terminator(b, Terminator::Ret(Ret::one(r))); @@ -98,17 +177,255 @@ fn narrow_store_keeps_neighbor_bytes() { assert_eq!(bytes, [0xab, 0x22, 0x33, 0x44, 0x55]); } #[test] -fn signed_conversion_and_float_nan_compare() { +fn big_endian_loads_are_byte_swapped() { + let bytes: [u8; 8] = [0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef]; + for (kind, expected) in [ + ("i16", u16::from_be_bytes([bytes[0], bytes[1]]) as u64), + ( + "i32", + u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) as u64, + ), + ("i64", u64::from_be_bytes(bytes)), + ] { + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, kind); + let wide = ty(&mut f, "i64"); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let mem = MemFlags { + endian: volt_ir::attribute::Endianness::Big, + ..MemFlags::new() + }; + let mut v = f.append_load(b, t, ptr, mem); + if t != wide { + v = f.append_inst( + b, + wide, + Opcode::Convert(Convert { + value: v, + kind: ConvertKind::Zext, + }), + ); + } + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*const u8) -> u64 = + unsafe { core::mem::transmute(code.ptr) }; + assert_eq!(unsafe { entry(bytes.as_ptr()) }, expected, "{kind}"); + } +} +/// `(p: ptr) -> i64` loading a `kind` integer `offset` bytes past `p` with `mem` and widening +/// it: sign-extending when `signed`, zero-extending otherwise (a load itself never extends +/// with the sign). +fn ordered_load(kind: &str, signed: bool, mem: MemFlags, offset: i64) -> Function { + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, kind); + let w = ty(&mut f, "i64"); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let addr = if offset == 0 { + ptr + } else { + f.append_arith_imm(b, p, BinOp::Add, ptr, offset) + }; + let mut v = f.append_load(b, t, addr, mem); + if t != w { + v = f.append_inst( + b, + w, + Opcode::Convert(Convert { + value: v, + kind: if signed { + ConvertKind::Sext + } else { + ConvertKind::Zext + }, + }), + ); + } + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + f +} +/// `(p: ptr, x: i64)` storing the low bits of `x` as a `kind` integer `offset` bytes past `p`. +fn ordered_store(kind: &str, mem: MemFlags, offset: i64) -> Function { let mut f = Function::new(); - let i = ty(&mut f, "i32"); - let o = ty(&mut f, "i64"); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, kind); + let w = ty(&mut f, "i64"); let b = f.append_block(); - let v = f.append_block_param(b, i); - let r = f.append_inst(b, o, Opcode::Convert(Convert { value: v })); - f.set_terminator(b, Terminator::Ret(Ret::one(r))); - let mem = Executable::new(&f); - let entry: unsafe extern "C" fn(i32) -> i64 = unsafe { core::mem::transmute(mem.ptr) }; - assert_eq!(unsafe { entry(-42) }, -42); + let ptr = f.append_block_param(b, p); + let x = f.append_block_param(b, w); + let narrow = if t == w { + x + } else { + f.append_inst( + b, + t, + Opcode::Convert(Convert { + value: x, + kind: ConvertKind::Trunc, + }), + ) + }; + let addr = if offset == 0 { + ptr + } else { + f.append_arith_imm(b, p, BinOp::Add, ptr, offset) + }; + f.append_store_mem(b, narrow, addr, mem); + f.set_terminator(b, Terminator::Ret(Ret::none())); + f +} +fn ordering_mem(ordering: Option, bytes: u32) -> MemFlags { + MemFlags { + ordering, + align: bytes, + ..MemFlags::new() + } +} +const LOAD_ORDERINGS: [Option; 4] = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Acquire), + Some(AtomicOrdering::SeqCst), +]; +const STORE_ORDERINGS: [Option; 4] = [ + None, + Some(AtomicOrdering::Relaxed), + Some(AtomicOrdering::Release), + Some(AtomicOrdering::SeqCst), +]; +#[repr(align(16))] +struct Image([u8; 16]); +const IMAGE: Image = Image([ + 0x81, 0x92, 0xa3, 0xb4, 0xc5, 0xd6, 0xe7, 0xf8, 0x09, 0x1a, 0x2b, 0x3c, 0x4d, 0x5e, 0x6f, 0x70, +]); +/// Every width and ordering loads the same value an ordinary load would, zero- or +/// sign-extended as the explicit conversion says. +#[test] +fn ordered_loads_produce_the_plain_load_value() { + let image = IMAGE; + for bits in [8u32, 16, 32, 64] { + for signed in [false, true] { + if signed && bits == 64 { + continue; + } + for offset in [0usize, 8] { + let n = (bits / 8) as usize; + let mut raw = [0u8; 8]; + raw[..n].copy_from_slice(&image.0[offset..offset + n]); + let zero = u64::from_le_bytes(raw); + let expected = if signed { + (((zero << (64 - bits)) as i64) >> (64 - bits)) as u64 + } else { + zero + }; + for ordering in LOAD_ORDERINGS { + let f = ordered_load( + &format!("i{bits}"), + signed, + ordering_mem(ordering, bits / 8), + offset as i64, + ); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*const u8) -> u64 = + unsafe { core::mem::transmute(code.ptr) }; + assert_eq!( + unsafe { entry(image.0.as_ptr()) }, + expected, + "i{bits} signed={signed} offset={offset} {ordering:?}" + ); + } + } + } + } +} +/// Every width and ordering (including the `xchg` of a sequentially consistent store) writes +/// exactly its own bytes. +#[test] +fn ordered_stores_write_exactly_their_width() { + let x = 0x0123_4567_89ab_cdefu64; + for (kind, bits) in [("i8", 8u32), ("i16", 16), ("i32", 32), ("i64", 64)] { + for offset in [0usize, 8] { + let n = (bits / 8) as usize; + for ordering in STORE_ORDERINGS { + let f = ordered_store(kind, ordering_mem(ordering, bits / 8), offset as i64); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*mut u8, u64) = + unsafe { core::mem::transmute(code.ptr) }; + let mut out = Image([0xee; 16]); + unsafe { entry(out.0.as_mut_ptr(), x) }; + let mut want = [0xeeu8; 16]; + want[offset..offset + n].copy_from_slice(&x.to_le_bytes()[..n]); + assert_eq!(out.0, want, "{kind} offset={offset} {ordering:?}"); + } + } + } +} +/// Float and pointer accesses: atomic floats go through an integer register and pointers are +/// 64-bit integers. +#[test] +fn ordered_float_and_pointer_accesses_execute() { + for ordering in [AtomicOrdering::Acquire, AtomicOrdering::SeqCst] { + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, "f64"); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let v = f.append_load(b, t, ptr, ordering_mem(Some(ordering), 8)); + f.set_terminator(b, Terminator::Ret(Ret::one(v))); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*const f64) -> f64 = + unsafe { core::mem::transmute(code.ptr) }; + let x = 6.25f64; + assert_eq!(unsafe { entry(&x) }, 6.25); + } + for ordering in [AtomicOrdering::Release, AtomicOrdering::SeqCst] { + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let t = ty(&mut f, "f32"); + let b = f.append_block(); + let ptr = f.append_block_param(b, p); + let x = f.append_block_param(b, t); + f.append_store_mem(b, x, ptr, ordering_mem(Some(ordering), 4)); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*mut f32, f32) = unsafe { core::mem::transmute(code.ptr) }; + let mut out = 0.0f32; + unsafe { entry(&mut out, -2.75) }; + assert_eq!(out, -2.75); + } + let mut f = Function::new(); + let p = ty(&mut f, "ptr"); + let b = f.append_block(); + let src = f.append_block_param(b, p); + let dst = f.append_block_param(b, p); + let v = f.append_load(b, p, src, ordering_mem(Some(AtomicOrdering::Acquire), 8)); + f.append_store_mem(b, v, dst, ordering_mem(Some(AtomicOrdering::SeqCst), 8)); + f.set_terminator(b, Terminator::Ret(Ret::none())); + let code = Executable::new(&f); + let entry: unsafe extern "C" fn(*const *const u8, *mut *const u8) = + unsafe { core::mem::transmute(code.ptr) }; + let target = 0x1234usize as *const u8; + let mut out = core::ptr::null(); + unsafe { entry(&target, &mut out) }; + assert_eq!(out, target); +} +#[test] +fn sign_and_zero_extension_differ() { + for (kind, want) in [ + (ConvertKind::Sext, -42i64 as u64), + (ConvertKind::Zext, 0xffff_ffd6), + ] { + let mem = Executable::new(&cases::convert("i32", "i64", kind)); + let entry: unsafe extern "C" fn(i32) -> u64 = unsafe { core::mem::transmute(mem.ptr) }; + assert_eq!(unsafe { entry(-42) }, want, "{kind:?}"); + } +} +#[test] +fn float_compare_with_nan_is_unordered() { for (cmp, want) in [ (CmpOp::Eq, 0), (CmpOp::Ne, 1), @@ -135,13 +452,13 @@ fn signed_conversion_and_float_nan_compare() { f.set_terminator(b, Terminator::Ret(Ret::one(v))); let mem = Executable::new(&f); let entry: unsafe extern "C" fn(f64, f64) -> u64 = unsafe { core::mem::transmute(mem.ptr) }; - assert_eq!(unsafe { entry(f64::NAN, 1.0) }, want); + assert_eq!(unsafe { entry(f64::NAN, 1.0) }, want, "{cmp:?}"); } } #[test] fn cfg_edge_parameter_cycles() { let mut f = Function::new(); - let t = ty(&mut f, "u64"); + let t = ty(&mut f, "i64"); let boolean = ty(&mut f, "bool"); let entry = f.append_block(); let loop_b = f.append_block(); @@ -193,7 +510,7 @@ fn vector_reduce_expansion() { v, Opcode::Load(Load { ptr, - volatile: false, + mem: MemFlags::new(), }), ); let r = f.append_reduce(b, BinOp::Add, loaded); @@ -208,7 +525,7 @@ fn indirect_call_preserves_live_values() { x * 3 } let mut f = Function::new(); - let t = ty(&mut f, "u64"); + let t = ty(&mut f, "i64"); let p = ty(&mut f, "ptr"); let b = f.append_block(); let x = f.append_block_param(b, t); @@ -242,18 +559,70 @@ fn indirect_call_preserves_live_values() { let entry: unsafe extern "C" fn(u64) -> u64 = unsafe { core::mem::transmute(mem.ptr) }; assert_eq!(unsafe { entry(13) }, 52); } +core::arch::global_asm!( + ".globl volt_test_dirty_return", + "volt_test_dirty_return:", + // al = 0x80, everything above it set: the System V ABI leaves those bits undefined. + "mov rax, -128", + "ret", +); +unsafe extern "C" { + fn volt_test_dirty_return() -> u8; +} +/// The bits above a narrow integer a callee returns are undefined, so a call result is +/// canonicalized (zero-extended from its declared width) before anything reads it. +#[test] +fn narrow_call_results_are_zero_extended_from_their_width() { + for (kind, convert, want) in [ + (ConvertKind::Zext, "i64", 0x80u64), + (ConvertKind::Sext, "i64", (-128i64) as u64), + (ConvertKind::Zext, "i32", 0x80), + (ConvertKind::Sext, "i32", 0xffff_ff80), + ] { + let mut f = Function::new(); + let byte = ty(&mut f, "i8"); + let wide = ty(&mut f, convert); + let p = ty(&mut f, "ptr"); + let b = f.append_block(); + let target = f.append_inst( + b, + p, + Opcode::Iconst(volt_test_dirty_return as *const () as usize as i64), + ); + let args = f.intern_values(&[]); + let r = f.append_inst( + b, + byte, + Opcode::CallIndirect(CallIndirect { + target, + args, + is_variadic: false, + num_fixed: 0, + ret_dest: None, + ret_regs: 0, + ret_pieces: [RetPiece::default(); 4], + sret: false, + }), + ); + let w = f.append_inst(b, wide, Opcode::Convert(Convert { value: r, kind })); + f.set_terminator(b, Terminator::Ret(Ret::one(w))); + let mem = Executable::new(&f); + let entry: unsafe extern "C" fn() -> u64 = unsafe { core::mem::transmute(mem.ptr) }; + assert_eq!(unsafe { entry() }, want, "{kind:?} to {convert}"); + } +} #[test] fn softfp_relocation_owns_injected_symbol() { - let f = binary("f128", BinOp::Add); + let f = cases::arith("f128", BinOp::Add); let out = isel::compile_object(&f).unwrap(); assert!(out.relocs.iter().any(|r| r.symbol == "__addtf3")); assert_eq!(f.symbol_count(), 0); } #[test] -fn narrow_arithmetic_wraps_and_compares_signed_values() { +fn narrow_arithmetic_wraps_and_compares_by_operation() { let mut f = Function::new(); - let byte = ty(&mut f, "u8"); + let byte = ty(&mut f, "i8"); let boolean = ty(&mut f, "bool"); let block = f.append_block(); let input = f.append_block_param(block, byte); @@ -283,46 +652,62 @@ fn narrow_arithmetic_wraps_and_compares_signed_values() { assert_eq!(unsafe { entry(255) }, 1); assert_eq!(unsafe { entry(1) }, 0); - let mut f = Function::new(); - let signed = ty(&mut f, "i8"); - let boolean = ty(&mut f, "bool"); - let block = f.append_block(); - let minus_one = f.append_inst(block, signed, Opcode::Iconst(-1)); - let zero = f.append_inst(block, signed, Opcode::Iconst(0)); - let negative = f.append_inst( - block, - boolean, - Opcode::Icmp(Compare { - op: CmpOp::Lt, - lhs: minus_one, - rhs: zero, - }), - ); - f.set_terminator(block, Terminator::Ret(Ret::one(negative))); - let memory = Executable::new(&f); - let entry: unsafe extern "C" fn() -> u64 = unsafe { core::mem::transmute(memory.ptr) }; - assert_eq!(unsafe { entry() }, 1); + // 0xff is -1 as a signed byte (less than 0) and 255 as an unsigned one (not less). + for (op, want) in [ + (CmpOp::Slt, 1), + (CmpOp::Ult, 0), + (CmpOp::Sgt, 0), + (CmpOp::Ugt, 1), + ] { + let mut f = Function::new(); + let byte = ty(&mut f, "i8"); + let boolean = ty(&mut f, "bool"); + let block = f.append_block(); + let minus_one = f.append_inst(block, byte, Opcode::Iconst(-1)); + let zero = f.append_inst(block, byte, Opcode::Iconst(0)); + let result = f.append_inst( + block, + boolean, + Opcode::Icmp(Compare { + op, + lhs: minus_one, + rhs: zero, + }), + ); + f.set_terminator(block, Terminator::Ret(Ret::one(result))); + let memory = Executable::new(&f); + let entry: unsafe extern "C" fn() -> u64 = unsafe { core::mem::transmute(memory.ptr) }; + assert_eq!(unsafe { entry() }, want, "{op:?}"); + } } #[test] fn high_product_uses_the_declared_integer_width() { - for (kind, a, b, expected) in [ - ("u8", 255, 255, 254), - ("u16", 65535, 65535, 65534), - ("u48", (1u64 << 48) - 1, (1u64 << 48) - 1, (1u64 << 48) - 2), - ("i8", (-2i64) as u64, 3, 255), + for (kind, op, a, b, expected) in [ + ("i8", BinOp::UMulh, 255, 255, 254), + ("i16", BinOp::UMulh, 65535, 65535, 65534), + ( + "i48", + BinOp::UMulh, + (1u64 << 48) - 1, + (1u64 << 48) - 1, + (1u64 << 48) - 2, + ), + // 0xfe is -2: the signed high byte of -2 * 3 = -6 is -1, the unsigned one of 254 * 3 = 762 + // is 2. + ("i8", BinOp::SMulh, 0xfe, 3, 0xff), + ("i8", BinOp::UMulh, 0xfe, 3, 2), + ("i16", BinOp::SMulh, 0xfffe, 3, 0xffff), + ("i16", BinOp::UMulh, 0xfffe, 3, 2), + ("i32", BinOp::SMulh, 0xffff_fffe, 3, 0xffff_ffff), + ("i32", BinOp::UMulh, 0xffff_fffe, 3, 2), + ("i48", BinOp::SMulh, (1u64 << 48) - 2, 3, (1u64 << 48) - 1), + ("i48", BinOp::UMulh, (1u64 << 48) - 2, 3, 2), ] { - let memory = Executable::new(&binary(kind, BinOp::Mulh)); + let memory = Executable::new(&cases::arith(kind, op)); let entry: unsafe extern "C" fn(u64, u64) -> u64 = unsafe { core::mem::transmute(memory.ptr) }; - let mask = if kind == "u48" { - (1u64 << 48) - 1 - } else if kind == "u16" { - 65535 - } else { - 255 - }; - assert_eq!(unsafe { entry(a, b) } & mask, expected, "{kind}"); + assert_eq!(unsafe { entry(a, b) }, expected, "{kind} {op:?}"); } } @@ -334,7 +719,7 @@ fn sret_returns_the_original_hidden_pointer_after_its_last_explicit_use() { let mut f = Function::new(); f.sret = true; let pointer = ty(&mut f, "ptr"); - let word = ty(&mut f, "u64"); + let word = ty(&mut f, "i64"); let block = f.append_block(); let destination = f.append_block_param(block, pointer); let value = f.append_inst(block, word, Opcode::Iconst(7));