javm_recompiler_x86/

asm.rs

//! x86-64 assembler for PVM recompiler.
//!
//! Emits native x86-64 machine code with label-based jump resolution.
//! All jumps use 32-bit relative offsets (no short-jump optimization).
//!
//! # Safety model
//!
//! The assembler writes to a raw `*mut u8` buffer (`self.buf`) for performance.
//! The key invariant: `self.buf` points to a valid allocation of at least
//! `self.capacity` bytes (the `code_buf` Vec's backing store).
//! All emission functions have `debug_assert!(self.write_pos + N <= self.capacity)`
//! guards. Callers must ensure capacity via `ensure_capacity()` before emitting.
//! Vec length is synced via `set_len(write_pos)` only at finalization boundaries.

use alloc::vec;
use alloc::vec::Vec;

/// Instruction buffer: accumulates x86 bytes in a u128 register, then flushes
/// with a single bulk write. Avoids per-byte memory stores.
#[derive(Clone, Copy)]
struct InstBuf {
    out: u128,
    length: u32, // in bits
}

impl InstBuf {
    #[inline(always)]
    fn new() -> Self {
        Self { out: 0, length: 0 }
    }

    #[inline(always)]
    fn push(&mut self, byte: u8) {
        self.out |= (byte as u128) << self.length;
        self.length += 8;
    }

    #[inline(always)]
    fn push_u32(&mut self, v: u32) {
        self.out |= (v as u128) << self.length;
        self.length += 32;
    }

    #[inline(always)]
    fn push_u64(&mut self, v: u64) {
        self.out |= (v as u128) << self.length;
        self.length += 64;
    }

    #[inline(always)]
    fn push_i32(&mut self, v: i32) {
        self.push_u32(v as u32);
    }

    #[inline(always)]
    fn len(&self) -> usize {
        (self.length >> 3) as usize
    }
}

/// x86-64 register encoding.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(u8)]
pub enum Reg {
    RAX = 0,
    RCX = 1,
    RDX = 2,
    RBX = 3,
    RSP = 4,
    RBP = 5,
    RSI = 6,
    RDI = 7,
    R8 = 8,
    R9 = 9,
    R10 = 10,
    R11 = 11,
    R12 = 12,
    R13 = 13,
    R14 = 14,
    R15 = 15,
}

impl Reg {
    /// Low 3 bits for ModR/M encoding.
    fn lo(self) -> u8 {
        (self as u8) & 7
    }
    /// High bit for REX.R or REX.B.
    fn hi(self) -> u8 {
        (self as u8) >> 3
    }
    /// Whether this register requires a REX prefix.
    fn needs_rex(self) -> bool {
        (self as u8) >= 8
    }
}

/// Condition codes for Jcc/SETcc/CMOVcc.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(u8)]
pub enum Cc {
    O = 0,
    NO = 1,
    B = 2,  // Below (unsigned <)
    AE = 3, // Above or Equal (unsigned >=)
    E = 4,  // Equal
    NE = 5, // Not Equal
    BE = 6, // Below or Equal (unsigned <=)
    A = 7,  // Above (unsigned >)
    S = 8,  // Sign
    NS = 9,
    P = 10,
    NP = 11,
    L = 12,  // Less (signed <)
    GE = 13, // Greater or Equal (signed >=)
    LE = 14, // Less or Equal (signed <=)
    G = 15,  // Greater (signed >)
}

/// Label identifier.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct Label(pub u32);

/// Fixup kind for label resolution.
#[derive(Clone, Copy)]
struct Fixup {
    /// Offset in code buffer where the 4-byte rel32 placeholder is.
    offset: usize,
    /// The label this fixup targets.
    label: Label,
}

/// x86-64 assembler with label support.
///
/// Uses direct pointer writes to the pre-allocated buffer for emission,
/// avoiding per-byte Vec::push overhead (capacity check + len update).
pub struct Assembler {
    code_buf: Vec<u8>,
    /// Raw pointer to the start of the code buffer.
    buf: *mut u8,
    write_pos: usize,
    capacity: usize,
    /// Label ID → bound offset+1 as u32 (0 = unbound). Uses u32 to halve
    /// memory vs usize (native code always fits in 4GB).
    /// Pre-sized via `vec![0u32; capacity]` which uses calloc (zero-page COW).
    labels: Vec<u32>,
    /// Number of labels allocated via new_label/bulk_create_labels.
    /// The Vec is pre-sized but labels_len tracks the logical length.
    labels_len: usize,
    fixups: Vec<Fixup>,
    /// Eventual VA the code buffer will be loaded at. Used by RIP-relative
    /// emitters to compute `disp32 = target_va - (jit_va_base + RIP_after_inst)`.
    /// Zero means "relocatable / unknown" — RIP-relative emitters then
    /// produce offsets relative to code-buffer offset 0, useful for tests.
    jit_va_base: u64,
}

/// Unbound label sentinel. We use 0 so that bulk label allocation can use
/// zeroed memory (calloc / zero-page COW) instead of writing 0xFF to every byte.
/// Bound labels store `native_offset + 1` to avoid collision with the sentinel.
const LABEL_UNBOUND: u32 = 0;

impl Default for Assembler {
    fn default() -> Self {
        Self::new()
    }
}

impl Assembler {
    pub fn new() -> Self {
        let mut code = Vec::with_capacity(4096);
        let buf = code.as_mut_ptr();
        let capacity = code.capacity();
        Self {
            code_buf: code,
            buf,
            write_pos: 0,
            capacity,
            labels: Vec::new(),
            labels_len: 0,
            fixups: Vec::new(),
            jit_va_base: 0,
        }
    }

    /// Create with pre-allocated capacity for code and labels.
    pub fn with_capacity(code_capacity: usize, label_capacity: usize) -> Self {
        let mut code = Vec::with_capacity(code_capacity);
        let buf = code.as_mut_ptr();
        let capacity = code.capacity();
        Self {
            code_buf: code,
            buf,
            write_pos: 0,
            capacity,
            // vec![0; n] uses calloc — zero pages via COW, no page faults for untouched entries
            labels: vec![0u32; label_capacity],
            labels_len: 0,
            fixups: Vec::with_capacity(2048),
            jit_va_base: 0,
        }
    }

    /// Set the eventual load VA for the code buffer. Must be called before
    /// any RIP-relative emitter that targets a fixed VA (i.e. CTX accesses).
    pub fn set_jit_va_base(&mut self, va: u64) {
        self.jit_va_base = va;
    }

    /// Ensure at least `additional` bytes of capacity remain.
    /// Called before emitting large sequences. Most individual instructions
    /// need at most ~32 bytes, so this is rarely needed mid-compilation.
    #[cold]
    fn grow(&mut self, additional: usize) {
        let code = &mut self.code_buf;
        // SAFETY: write_pos <= code.capacity() is maintained by ensure_capacity().
        // set_len(write_pos) exposes the bytes we've written so reserve() copies them.
        unsafe {
            code.set_len(self.write_pos);
        }
        code.reserve(additional);
        self.buf = code.as_mut_ptr();
        self.capacity = code.capacity();
        // SAFETY: Reset len to 0 — we track the actual length via write_pos,
        // not Vec::len. The data is preserved in the backing allocation.
        unsafe {
            code.set_len(0);
        }
    }

    /// Check capacity and grow if needed. Inlined for the fast path (no grow).
    #[inline(always)]
    pub fn ensure_capacity(&mut self, n: usize) {
        if self.write_pos + n > self.capacity {
            self.grow(n);
        }
    }

    /// Allocate a new label.
    pub fn new_label(&mut self) -> Label {
        let id = self.labels_len as u32;
        self.labels_len += 1;
        // Grow if needed (rare — labels Vec is pre-sized in with_capacity)
        if self.labels_len > self.labels.len() {
            self.labels.push(LABEL_UNBOUND);
        }
        Label(id)
    }

    /// Current number of labels allocated.
    pub fn labels_len(&self) -> usize {
        self.labels_len
    }

    /// Bulk-allocate `count` unbound labels. The labels Vec is already pre-sized
    /// via calloc (zero pages). This just advances the logical length counter.
    pub fn bulk_create_labels(&mut self, count: usize) {
        self.labels_len += count;
        // Grow if pre-sized Vec wasn't large enough (shouldn't happen normally)
        if self.labels_len > self.labels.len() {
            self.labels.resize(self.labels_len, LABEL_UNBOUND);
        }
    }

    /// Bind a label to the current write position.
    pub fn bind_label(&mut self, label: Label) {
        self.labels[label.0 as usize] = (self.write_pos + 1) as u32; // +1: 0 is LABEL_UNBOUND
    }

    /// Current code offset (write position).
    pub fn offset(&self) -> usize {
        self.write_pos
    }

    /// Patch an i32 value at a previously recorded offset.
    pub fn patch_i32(&mut self, offset: usize, value: i32) {
        debug_assert!(offset + 4 <= self.write_pos);
        // SAFETY: offset + 4 <= write_pos <= capacity, so buf.add(offset) is in bounds.
        unsafe {
            core::ptr::copy_nonoverlapping(value.to_le_bytes().as_ptr(), self.buf.add(offset), 4);
        }
    }

    // === Raw byte emission ===
    // All emission writes directly to the buffer via raw pointer,
    // bypassing Vec::push's capacity check and len update.
    // SAFETY for all emit* functions: write_pos + N <= capacity is guarded by
    // debug_assert. Callers ensure capacity via ensure_capacity() before sequences.

    #[inline(always)]
    fn emit(&mut self, b: u8) {
        debug_assert!(self.write_pos < self.capacity);
        // SAFETY: write_pos < capacity is asserted above; buf points to a valid
        // allocation of at least `capacity` bytes (Vec backing store).
        unsafe {
            *self.buf.add(self.write_pos) = b;
        }
        self.write_pos += 1;
    }

    /// Emit 3 bytes at once.
    #[inline(always)]
    fn emit3(&mut self, a: u8, b: u8, c: u8) {
        debug_assert!(self.write_pos + 3 <= self.capacity);
        // SAFETY: write_pos + 3 <= capacity is asserted above; buf is a valid
        // allocation. Individual byte writes are in-bounds.
        unsafe {
            let p = self.buf.add(self.write_pos);
            *p = a;
            *p.add(1) = b;
            *p.add(2) = c;
        }
        self.write_pos += 3;
    }

    /// Flush an InstBuf to the code buffer in one bulk write.
    #[inline(always)]
    fn flush_instbuf(&mut self, ib: InstBuf) {
        let len = ib.len();
        debug_assert!(self.write_pos + len <= self.capacity);
        // SAFETY: write_pos + len <= capacity is asserted above. We always
        // write both 8-byte halves (16 bytes total) even if len < 16 — this
        // is safe because ensure_capacity(512) guarantees ample slack beyond
        // the actual instruction bytes.
        unsafe {
            let p = self.buf.add(self.write_pos);
            core::ptr::write_unaligned(p as *mut u64, ib.out as u64);
            core::ptr::write_unaligned(p.add(8) as *mut u64, (ib.out >> 64) as u64);
        }
        self.write_pos += len;
    }

    #[inline(always)]
    fn emit_u32(&mut self, v: u32) {
        debug_assert!(self.write_pos + 4 <= self.capacity);
        // SAFETY: write_pos + 4 <= capacity asserted; unaligned write is valid
        // for any byte-aligned pointer within the buffer.
        unsafe {
            core::ptr::write_unaligned(self.buf.add(self.write_pos) as *mut u32, v.to_le());
        }
        self.write_pos += 4;
    }

    #[inline(always)]
    fn emit_i32(&mut self, v: i32) {
        debug_assert!(self.write_pos + 4 <= self.capacity);
        // SAFETY: write_pos + 4 <= capacity asserted; unaligned write is valid.
        // The cast chain converts i32 → LE bytes → u32 for write_unaligned.
        unsafe {
            core::ptr::write_unaligned(
                self.buf.add(self.write_pos) as *mut u32,
                v.to_le_bytes().as_ptr().cast::<u32>().read(),
            );
        }
        self.write_pos += 4;
    }

    /// Emit a label reference (4-byte rel32). For backward references (label
    /// already bound), resolves immediately without creating a fixup entry.
    /// For forward references, emits a placeholder and records a fixup.
    fn emit_label_fixup(&mut self, label: Label) {
        let bound = self.labels[label.0 as usize];
        if bound != LABEL_UNBOUND {
            // Backward reference — resolve immediately, no fixup needed.
            // rel32 = target - (current_offset + 4). Stored value is offset+1.
            let target = (bound - 1) as i64;
            let rel = target - (self.write_pos as i64 + 4);
            self.emit_i32(rel as i32);
        } else {
            // Forward reference — defer to finalization.
            let offset = self.write_pos;
            self.fixups.push(Fixup { offset, label });
            self.emit_u32(0); // placeholder
        }
    }

    // === REX prefix helpers ===

    fn rex_opt_b(&mut self, rm: Reg) {
        if rm.needs_rex() {
            self.emit(0x40 | rm.hi());
        }
    }

    /// ModR/M (+ optional SIB) + displacement for [base + disp] addressing.
    /// Pushes into an InstBuf instead of emitting directly.
    #[inline(always)]
    fn modrm_disp_ib(ib: &mut InstBuf, reg: u8, base: Reg, disp: i32) {
        let bl = base.lo();
        let needs_sib = bl == 4;

        if disp == 0 && bl != 5 {
            if needs_sib {
                ib.push((reg << 3) | 4);
                ib.push(0x24);
            } else {
                ib.push((reg << 3) | bl);
            }
        } else if (-128..=127).contains(&disp) {
            if needs_sib {
                ib.push(0x40 | (reg << 3) | 4);
                ib.push(0x24);
            } else {
                ib.push(0x40 | (reg << 3) | bl);
            }
            ib.push(disp as u8);
        } else {
            if needs_sib {
                ib.push(0x80 | (reg << 3) | 4);
                ib.push(0x24);
            } else {
                ib.push(0x80 | (reg << 3) | bl);
            }
            ib.push_i32(disp);
        }
    }

    // === Instruction emission ===

    // -- MOV --

    /// mov r64, r64
    #[inline(always)]
    pub fn mov_rr(&mut self, dst: Reg, src: Reg) {
        if dst == src {
            return;
        }
        self.emit3(
            0x48 | (src.hi() << 2) | dst.hi(),
            0x89,
            0xC0 | (src.lo() << 3) | dst.lo(),
        );
    }

    /// mov r64, imm64
    #[inline(always)]
    pub fn mov_ri64(&mut self, dst: Reg, imm: u64) {
        let mut ib = InstBuf::new();
        if imm == 0 {
            // xor r32, r32 (clears full r64)
            let r = dst.hi();
            if r != 0 {
                ib.push(0x40 | (r << 2) | r);
            }
            ib.push(0x31);
            ib.push(0xC0 | (dst.lo() << 3) | dst.lo());
        } else if imm <= u32::MAX as u64 {
            // mov r32, imm32 (zero-extends to 64)
            if dst.needs_rex() {
                ib.push(0x40 | dst.hi());
            }
            ib.push(0xB8 + dst.lo());
            ib.push_u32(imm as u32);
        } else if imm as i64 >= i32::MIN as i64 && imm as i64 <= i32::MAX as i64 {
            // mov r64, sign-extended imm32
            ib.push(0x48 | dst.hi());
            ib.push(0xC7);
            ib.push(0xC0 | dst.lo());
            ib.push_i32(imm as i32);
        } else {
            // mov r64, imm64
            ib.push(0x48 | dst.hi());
            ib.push(0xB8 + dst.lo());
            ib.push_u64(imm);
        }
        self.flush_instbuf(ib);
    }

    /// mov r32, imm32 (zero-extends to 64-bit)
    #[inline(always)]
    pub fn mov_ri32(&mut self, dst: Reg, imm: u32) {
        let mut ib = InstBuf::new();
        if dst.needs_rex() {
            ib.push(0x40 | dst.hi());
        }
        ib.push(0xB8 + dst.lo());
        ib.push_u32(imm);
        self.flush_instbuf(ib);
    }

    /// mov r64, [base + disp]
    #[inline(always)]
    pub fn mov_load64(&mut self, dst: Reg, base: Reg, disp: i32) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (dst.hi() << 2) | base.hi());
        ib.push(0x8B);
        Self::modrm_disp_ib(&mut ib, dst.lo(), base, disp);
        self.flush_instbuf(ib);
    }

    /// movsxd r64, dword [base + index*4] — sign-extending load with SIB scale=4
    pub fn movsxd_load_sib4(&mut self, dst: Reg, base: Reg, index: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (dst.hi() << 2) | (index.hi() << 1) | base.hi());
        ib.push(0x63);
        ib.push((dst.lo() << 3) | 4);
        ib.push(0x80 | (index.lo() << 3) | base.lo());
        self.flush_instbuf(ib);
    }

    /// mov dword [base + disp], r32 — 32-bit store
    #[inline(always)]
    pub fn mov_store32(&mut self, base: Reg, disp: i32, src: Reg) {
        let mut ib = InstBuf::new();
        let r = src.hi();
        let b = base.hi();
        if r != 0 || b != 0 {
            ib.push(0x40 | (r << 2) | b);
        }
        ib.push(0x89);
        Self::modrm_disp_ib(&mut ib, src.lo(), base, disp);
        self.flush_instbuf(ib);
    }

    /// mov [base + disp], r64
    #[inline(always)]
    pub fn mov_store64(&mut self, base: Reg, disp: i32, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (src.hi() << 2) | base.hi());
        ib.push(0x89);
        Self::modrm_disp_ib(&mut ib, src.lo(), base, disp);
        self.flush_instbuf(ib);
    }

    // -- ALU reg,reg (64-bit) --

    fn alu_rr64(&mut self, op: u8, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (src.hi() << 2) | dst.hi());
        ib.push(op);
        ib.push(0xC0 | (src.lo() << 3) | dst.lo());
        self.flush_instbuf(ib);
    }

    fn alu_rr32(&mut self, op: u8, dst: Reg, src: Reg) {
        let r = src.hi();
        let b = dst.hi();
        if r != 0 || b != 0 {
            let mut ib = InstBuf::new();
            ib.push(0x40 | (r << 2) | b);
            ib.push(op);
            ib.push(0xC0 | (src.lo() << 3) | dst.lo());
            self.flush_instbuf(ib);
        } else {
            let mut ib = InstBuf::new();
            ib.push(op);
            ib.push(0xC0 | (src.lo() << 3) | dst.lo());
            self.flush_instbuf(ib);
        }
    }

    #[inline(always)]
    pub fn add_rr(&mut self, dst: Reg, src: Reg) {
        self.alu_rr64(0x01, dst, src);
    }
    #[inline(always)]
    pub fn sub_rr(&mut self, dst: Reg, src: Reg) {
        self.alu_rr64(0x29, dst, src);
    }
    #[inline(always)]
    pub fn and_rr(&mut self, dst: Reg, src: Reg) {
        self.alu_rr64(0x21, dst, src);
    }
    #[inline(always)]
    pub fn or_rr(&mut self, dst: Reg, src: Reg) {
        self.alu_rr64(0x09, dst, src);
    }
    #[inline(always)]
    pub fn xor_rr(&mut self, dst: Reg, src: Reg) {
        self.alu_rr64(0x31, dst, src);
    }
    #[inline(always)]
    pub fn cmp_rr(&mut self, a: Reg, b: Reg) {
        self.alu_rr64(0x39, a, b);
    }
    #[inline(always)]
    pub fn test_rr(&mut self, a: Reg, b: Reg) {
        self.alu_rr64(0x85, a, b);
    }

    /// `test r32, r32` — sets ZF iff the low 32 bits are zero (ignores the
    /// high 32 bits). Used by the 32-bit `divw`/`remw` zero-divisor guard,
    /// which must test only the bits `idivl`/`divl` actually consume.
    #[inline(always)]
    pub fn test_rr32(&mut self, a: Reg, b: Reg) {
        self.alu_rr32(0x85, a, b);
    }

    #[inline(always)]
    pub fn add_rr32(&mut self, dst: Reg, src: Reg) {
        self.alu_rr32(0x01, dst, src);
    }
    #[inline(always)]
    pub fn sub_rr32(&mut self, dst: Reg, src: Reg) {
        self.alu_rr32(0x29, dst, src);
    }

    // -- ALU reg,imm (64-bit) --
    // Uses imm8 (opcode 0x83) when immediate fits in -128..127, saving 3 bytes.

    #[inline(always)]
    fn alu_ri64(&mut self, ext: u8, dst: Reg, imm: i32) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | dst.hi());
        if (-128..=127).contains(&imm) {
            ib.push(0x83);
            ib.push(0xC0 | (ext << 3) | dst.lo());
            ib.push(imm as u8);
        } else {
            ib.push(0x81);
            ib.push(0xC0 | (ext << 3) | dst.lo());
            ib.push_i32(imm);
        }
        self.flush_instbuf(ib);
    }

    #[inline(always)]
    fn alu_ri32(&mut self, ext: u8, dst: Reg, imm: i32) {
        let mut ib = InstBuf::new();
        if dst.needs_rex() {
            ib.push(0x40 | dst.hi());
        }
        if (-128..=127).contains(&imm) {
            ib.push(0x83);
            ib.push(0xC0 | (ext << 3) | dst.lo());
            ib.push(imm as u8);
        } else {
            ib.push(0x81);
            ib.push(0xC0 | (ext << 3) | dst.lo());
            ib.push_i32(imm);
        }
        self.flush_instbuf(ib);
    }

    #[inline(always)]
    pub fn add_ri(&mut self, dst: Reg, imm: i32) {
        self.alu_ri64(0, dst, imm);
    }
    #[inline(always)]
    pub fn sub_ri(&mut self, dst: Reg, imm: i32) {
        self.alu_ri64(5, dst, imm);
    }
    #[inline(always)]
    pub fn and_ri(&mut self, dst: Reg, imm: i32) {
        self.alu_ri64(4, dst, imm);
    }
    #[inline(always)]
    pub fn or_ri(&mut self, dst: Reg, imm: i32) {
        self.alu_ri64(1, dst, imm);
    }
    #[inline(always)]
    pub fn xor_ri(&mut self, dst: Reg, imm: i32) {
        self.alu_ri64(6, dst, imm);
    }
    #[inline(always)]
    pub fn cmp_ri(&mut self, a: Reg, imm: i32) {
        self.alu_ri64(7, a, imm);
    }

    #[inline(always)]
    pub fn add_ri32(&mut self, dst: Reg, imm: i32) {
        self.alu_ri32(0, dst, imm);
    }
    #[inline(always)]
    pub fn sub_ri32(&mut self, dst: Reg, imm: i32) {
        self.alu_ri32(5, dst, imm);
    }
    #[inline(always)]
    pub fn cmp_ri32(&mut self, a: Reg, imm: i32) {
        self.alu_ri32(7, a, imm);
    }

    // -- Baseless memory access [idx] (mod=00 r/m=idx, no SIB) --
    //
    // Codegen uses this when the PVM addr is itself the host VA — no
    // base register needed because the runtime substrate's per-
    // invocation page table maps `addr` → `addr` for the guest range.
    // Saves one byte per access vs the SIB form `[base + idx]`.
    //
    // ModRM bytes: `(mod=00, reg, r/m=idx.lo())` if idx.lo() ∉ {4, 5}.
    // For `idx.lo() == 4` (RSP/R12) the ModRM r/m=100 encoding means
    // "SIB follows"; emit SIB = `(scale=00, index=100, base=4)` to
    // recover plain `[idx]`. For `idx.lo() == 5` (RBP/R13) the
    // mod=00 r/m=101 form is reserved for [RIP+disp32], so use
    // mod=01 + disp8=0.
    fn modrm_baseless(&mut self, reg: u8, idx: Reg) {
        let r = reg & 7;
        match idx.lo() {
            4 => {
                // SIB form: [RSP/R12]
                self.emit((r << 3) | 4); // mod=00 r/m=100
                self.emit(0x24); // SIB: scale=00, index=100=none, base=4
            }
            5 => {
                // mod=01 disp8=0: [RBP/R13]
                self.emit(0x40 | (r << 3) | 5);
                self.emit(0);
            }
            _ => {
                self.emit((r << 3) | idx.lo());
            }
        }
    }

    /// movzx r64, byte \[idx\] — zero-extending u8 load
    pub fn movzx_load8_at_index(&mut self, dst: Reg, idx: Reg) {
        self.emit(0x48 | (dst.hi() << 2) | idx.hi());
        self.emit(0x0F);
        self.emit(0xB6);
        self.modrm_baseless(dst.lo(), idx);
    }

    /// movzx r64, word \[idx\] — zero-extending u16 load
    pub fn movzx_load16_at_index(&mut self, dst: Reg, idx: Reg) {
        let rex = 0x40 | (dst.hi() << 2) | idx.hi();
        if rex != 0x40 {
            self.emit(rex);
        }
        self.emit(0x0F);
        self.emit(0xB7);
        self.modrm_baseless(dst.lo(), idx);
    }

    /// mov r32, dword \[idx\] — zero-extending u32 load (writes EAX, clears upper 32)
    pub fn mov_load32_at_index(&mut self, dst: Reg, idx: Reg) {
        let rex = 0x40 | (dst.hi() << 2) | idx.hi();
        if rex != 0x40 {
            self.emit(rex);
        }
        self.emit(0x8B);
        self.modrm_baseless(dst.lo(), idx);
    }

    /// mov r64, qword \[idx\]
    pub fn mov_load64_at_index(&mut self, dst: Reg, idx: Reg) {
        self.emit(0x48 | (dst.hi() << 2) | idx.hi());
        self.emit(0x8B);
        self.modrm_baseless(dst.lo(), idx);
    }

    /// mov byte \[idx\], r8
    pub fn mov_store8_at_index(&mut self, idx: Reg, src: Reg) {
        // REX prefix mandatory if `src` is SIL/DIL/BPL/SPL (encoded
        // 4..=7 with hi=0) — without REX those encodings decode as
        // AH/CH/DH/BH. (`needs_rex` only catches R8-R15.)
        let rex = 0x40 | (src.hi() << 2) | idx.hi();
        if rex != 0x40 || src.lo() >= 4 {
            self.emit(rex);
        }
        self.emit(0x88);
        self.modrm_baseless(src.lo(), idx);
    }

    /// mov word \[idx\], r16
    pub fn mov_store16_at_index(&mut self, idx: Reg, src: Reg) {
        self.emit(0x66);
        let rex = 0x40 | (src.hi() << 2) | idx.hi();
        if rex != 0x40 {
            self.emit(rex);
        }
        self.emit(0x89);
        self.modrm_baseless(src.lo(), idx);
    }

    /// mov dword \[idx\], r32
    pub fn mov_store32_at_index(&mut self, idx: Reg, src: Reg) {
        let rex = 0x40 | (src.hi() << 2) | idx.hi();
        if rex != 0x40 {
            self.emit(rex);
        }
        self.emit(0x89);
        self.modrm_baseless(src.lo(), idx);
    }

    /// mov qword \[idx\], r64
    pub fn mov_store64_at_index(&mut self, idx: Reg, src: Reg) {
        self.emit(0x48 | (src.hi() << 2) | idx.hi());
        self.emit(0x89);
        self.modrm_baseless(src.lo(), idx);
    }

    // -- RIP-relative addressing `[rip + disp32]` --
    //
    // 64-bit mode's mod=00 r/m=101 form is RIP-relative: the effective
    // address is `RIP_after_instruction + sign_extend(disp32)`. Used to
    // reach CTX, which lives at 512 GiB (PML4 slot 1, adjacent to the
    // JIT arena), outside the guest u32 address space and therefore
    // beyond absolute-SIB disp32 reach. The `jit_va_base` field of the
    // assembler holds the eventual load VA;
    // `target_va - (jit_va_base + write_pos + 4)` is the disp32 we
    // emit.
    //
    // 1 byte shorter per access than the absolute-SIB form (no SIB
    // byte). Range: target must be within ±2 GiB of post-instruction
    // RIP, which holds because CTX sits adjacent to the JIT arena.
    /// Emit a 4-byte RIP-relative disp32 targeting `target_va`. The CPU
    /// resolves the effective address against RIP-at-next-instruction,
    /// so we add 4 (for the disp32 itself) plus `trailing_bytes` (any
    /// imm fields following disp32 in the same instruction — 4 for
    /// `mov [rip+disp32], imm32`, 0 for everything else).
    fn emit_rip_rel_disp32(&mut self, target_va: u64, trailing_bytes: u64) {
        let post_inst_rip = self
            .jit_va_base
            .wrapping_add(self.write_pos as u64)
            .wrapping_add(4)
            .wrapping_add(trailing_bytes);
        let disp = (target_va as i64).wrapping_sub(post_inst_rip as i64);
        debug_assert!(
            disp >= i32::MIN as i64 && disp <= i32::MAX as i64,
            "RIP-relative target 0x{:x} out of range from base 0x{:x} + offset 0x{:x}",
            target_va,
            self.jit_va_base,
            self.write_pos
        );
        self.emit_i32(disp as i32);
    }

    fn modrm_rip_rel(&mut self, reg: u8) {
        // mod=00, reg, r/m=101 = RIP-relative (no SIB byte).
        self.emit(((reg & 7) << 3) | 5);
    }

    /// mov r32, dword [rip+disp32] (zero-extends to 64-bit)
    pub fn mov_load32_rip_rel(&mut self, dst: Reg, target_va: u64) {
        if dst.hi() != 0 {
            self.emit(0x40 | (dst.hi() << 2));
        }
        self.emit(0x8B);
        self.modrm_rip_rel(dst.lo());
        self.emit_rip_rel_disp32(target_va, 0);
    }

    /// mov r64, qword [rip+disp32]
    pub fn mov_load64_rip_rel(&mut self, dst: Reg, target_va: u64) {
        self.emit(0x48 | (dst.hi() << 2));
        self.emit(0x8B);
        self.modrm_rip_rel(dst.lo());
        self.emit_rip_rel_disp32(target_va, 0);
    }

    /// mov dword [rip+disp32], r32
    pub fn mov_store32_rip_rel(&mut self, target_va: u64, src: Reg) {
        if src.hi() != 0 {
            self.emit(0x40 | (src.hi() << 2));
        }
        self.emit(0x89);
        self.modrm_rip_rel(src.lo());
        self.emit_rip_rel_disp32(target_va, 0);
    }

    /// mov qword [rip+disp32], r64
    pub fn mov_store64_rip_rel(&mut self, target_va: u64, src: Reg) {
        self.emit(0x48 | (src.hi() << 2));
        self.emit(0x89);
        self.modrm_rip_rel(src.lo());
        self.emit_rip_rel_disp32(target_va, 0);
    }

    /// mov dword [rip+disp32], imm32 — the trailing imm32 means the
    /// "next-instruction RIP" is 4 bytes further than disp32 alone.
    pub fn mov_store32_rip_rel_imm(&mut self, target_va: u64, imm: i32) {
        self.emit(0xC7);
        self.modrm_rip_rel(0);
        self.emit_rip_rel_disp32(target_va, 4);
        self.emit_i32(imm);
    }

    /// add r64, qword [rip+disp32]
    pub fn add_r64_mem_rip_rel(&mut self, dst: Reg, target_va: u64) {
        self.emit(0x48 | (dst.hi() << 2));
        self.emit(0x03);
        self.modrm_rip_rel(dst.lo());
        self.emit_rip_rel_disp32(target_va, 0);
    }

    /// cmp r64, qword [rip+disp32] — compare a register against a memory
    /// operand (used by the x3/x4 spill path's branch, where a spilled
    /// source lives in `JitContext.regs[13|14]`).
    pub fn cmp_r64_mem_rip_rel(&mut self, dst: Reg, target_va: u64) {
        self.emit(0x48 | (dst.hi() << 2));
        self.emit(0x3B);
        self.modrm_rip_rel(dst.lo());
        self.emit_rip_rel_disp32(target_va, 0);
    }

    // -- In-register gas decrement (patchable) --

    /// sub r64, imm32 in the always-imm32 (7-byte) encoding.
    /// `offset() - 4` after this call points at the imm32 field;
    /// callers patch it after emission for per-block gas metering.
    pub fn sub_r64_imm32_patchable(&mut self, dst: Reg, imm: i32) {
        // NOTE: Cannot use InstBuf here — caller reads offset() for
        // patching. The imm32 must be the trailing 4 bytes.
        self.emit(0x48 | dst.hi());
        self.emit(0x81);
        self.emit(0xE8 | dst.lo()); // mod=11 (register), reg=5 (sub), r/m=dst.lo()
        self.emit_i32(imm);
    }

    /// cmp r64, imm32 in the always-imm32 (7-byte) encoding — a
    /// non-mutating gate (sets flags, leaves `dst` unchanged). Mirrors
    /// [`Self::sub_r64_imm32_patchable`] but with reg-field = 7 (CMP).
    /// `offset() - 4` after this call points at the imm32 field; callers
    /// patch it for the per-block check-before-charge gas gate.
    pub fn cmp_r64_imm32_patchable(&mut self, dst: Reg, imm: i32) {
        self.emit(0x48 | dst.hi());
        self.emit(0x81);
        self.emit(0xF8 | dst.lo()); // mod=11 (register), reg=7 (cmp), r/m=dst.lo()
        self.emit_i32(imm);
    }

    // -- IMUL --

    /// imul r64, r64
    pub fn imul_rr(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (dst.hi() << 2) | src.hi());
        ib.push(0x0F);
        ib.push(0xAF);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// imul r32, r32
    pub fn imul_rr32(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        let r = dst.hi();
        let b = src.hi();
        if r != 0 || b != 0 {
            ib.push(0x40 | (r << 2) | b);
        }
        ib.push(0x0F);
        ib.push(0xAF);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    // -- MUL/IMUL widening (RDX:RAX = RAX * src) --

    /// mul r64 (unsigned RDX:RAX = RAX * src)
    pub fn mul_rdx_rax(&mut self, src: Reg) {
        self.emit3(0x48 | src.hi(), 0xF7, 0xE0 | src.lo());
    }

    /// imul r64 (signed RDX:RAX = RAX * src)
    pub fn imul_rdx_rax(&mut self, src: Reg) {
        self.emit3(0x48 | src.hi(), 0xF7, 0xE8 | src.lo());
    }

    // -- DIV/IDIV --

    /// div r64 (unsigned RAX = RDX:RAX / src, RDX = remainder)
    pub fn div64(&mut self, src: Reg) {
        self.emit3(0x48 | src.hi(), 0xF7, 0xF0 | src.lo());
    }

    /// idiv r64 (signed)
    pub fn idiv64(&mut self, src: Reg) {
        self.emit3(0x48 | src.hi(), 0xF7, 0xF8 | src.lo());
    }

    /// div r32
    pub fn div32(&mut self, src: Reg) {
        if src.needs_rex() {
            self.emit3(0x41, 0xF7, 0xF0 | src.lo());
        } else {
            let mut ib = InstBuf::new();
            ib.push(0xF7);
            ib.push(0xF0 | src.lo());
            self.flush_instbuf(ib);
        }
    }

    /// idiv r32
    pub fn idiv32(&mut self, src: Reg) {
        if src.needs_rex() {
            self.emit3(0x41, 0xF7, 0xF8 | src.lo());
        } else {
            let mut ib = InstBuf::new();
            ib.push(0xF7);
            ib.push(0xF8 | src.lo());
            self.flush_instbuf(ib);
        }
    }

    /// cqo (sign-extend RAX into RDX:RAX, 64-bit)
    pub fn cqo(&mut self) {
        self.emit(0x48);
        self.emit(0x99);
    }

    /// cdq (sign-extend EAX into EDX:EAX, 32-bit)
    pub fn cdq(&mut self) {
        self.emit(0x99);
    }

    // -- NEG/NOT --

    /// neg r64
    pub fn neg64(&mut self, dst: Reg) {
        self.emit3(0x48 | dst.hi(), 0xF7, 0xD8 | dst.lo());
    }

    /// not r64
    pub fn not64(&mut self, dst: Reg) {
        self.emit3(0x48 | dst.hi(), 0xF7, 0xD0 | dst.lo());
    }

    // -- Shifts --

    fn shift_ri64(&mut self, ext: u8, dst: Reg, imm: u8) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | dst.hi());
        ib.push(0xC1);
        ib.push(0xC0 | (ext << 3) | dst.lo());
        ib.push(imm);
        self.flush_instbuf(ib);
    }

    pub fn shift_cl64(&mut self, ext: u8, dst: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | dst.hi());
        ib.push(0xD3);
        ib.push(0xC0 | (ext << 3) | dst.lo());
        self.flush_instbuf(ib);
    }

    fn shift_ri32(&mut self, ext: u8, dst: Reg, imm: u8) {
        let mut ib = InstBuf::new();
        if dst.needs_rex() {
            ib.push(0x40 | dst.hi());
        }
        ib.push(0xC1);
        ib.push(0xC0 | (ext << 3) | dst.lo());
        ib.push(imm);
        self.flush_instbuf(ib);
    }

    pub fn shift_cl32(&mut self, ext: u8, dst: Reg) {
        let mut ib = InstBuf::new();
        if dst.needs_rex() {
            ib.push(0x40 | dst.hi());
        }
        ib.push(0xD3);
        ib.push(0xC0 | (ext << 3) | dst.lo());
        self.flush_instbuf(ib);
    }

    pub fn shl_ri64(&mut self, dst: Reg, imm: u8) {
        self.shift_ri64(4, dst, imm);
    }
    pub fn shr_ri64(&mut self, dst: Reg, imm: u8) {
        self.shift_ri64(5, dst, imm);
    }
    pub fn sar_ri64(&mut self, dst: Reg, imm: u8) {
        self.shift_ri64(7, dst, imm);
    }
    pub fn shl_cl64(&mut self, dst: Reg) {
        self.shift_cl64(4, dst);
    }
    pub fn ror_ri64(&mut self, dst: Reg, imm: u8) {
        self.shift_ri64(1, dst, imm);
    }

    pub fn shl_ri32(&mut self, dst: Reg, imm: u8) {
        self.shift_ri32(4, dst, imm);
    }
    pub fn shr_ri32(&mut self, dst: Reg, imm: u8) {
        self.shift_ri32(5, dst, imm);
    }
    pub fn sar_ri32(&mut self, dst: Reg, imm: u8) {
        self.shift_ri32(7, dst, imm);
    }
    pub fn ror_ri32(&mut self, dst: Reg, imm: u8) {
        self.shift_ri32(1, dst, imm);
    }

    // -- Extensions --

    /// movsxd r64, r32 (sign-extend 32→64)
    #[inline(always)]
    pub fn movsxd(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (dst.hi() << 2) | src.hi());
        ib.push(0x63);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// movsx r64, r8 (sign-extend 8→64)
    pub fn movsx_8_64(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (dst.hi() << 2) | src.hi());
        ib.push(0x0F);
        ib.push(0xBE);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// movsx r64, r16 (sign-extend 16→64)
    pub fn movsx_16_64(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (dst.hi() << 2) | src.hi());
        ib.push(0x0F);
        ib.push(0xBF);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// movzx r32, r16 (zero-extends to 64 due to 32-bit operation)
    pub fn movzx_16_64(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        let r = dst.hi();
        let b = src.hi();
        if r != 0 || b != 0 {
            ib.push(0x40 | (r << 2) | b);
        }
        ib.push(0x0F);
        ib.push(0xB7);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// Zero-extend 32→64: mov r32, r32 (implicit zero-extend)
    #[inline(always)]
    pub fn movzx_32_64(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        let r = src.hi();
        let b = dst.hi();
        if r != 0 || b != 0 {
            ib.push(0x40 | (r << 2) | b);
        }
        ib.push(0x89);
        ib.push(0xC0 | (src.lo() << 3) | dst.lo());
        self.flush_instbuf(ib);
    }

    // -- Conditional set --

    /// setcc r8 (sets low byte, need to movzx after)
    #[inline(always)]
    pub fn setcc(&mut self, cc: Cc, dst: Reg) {
        let mut ib = InstBuf::new();
        if dst.needs_rex() || matches!(dst, Reg::RSP | Reg::RBP | Reg::RSI | Reg::RDI) {
            ib.push(0x40 | dst.hi());
        }
        ib.push(0x0F);
        ib.push(0x90 + cc as u8);
        ib.push(0xC0 | dst.lo());
        self.flush_instbuf(ib);
    }

    /// cmovcc r64, r64
    #[inline(always)]
    pub fn cmovcc(&mut self, cc: Cc, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (dst.hi() << 2) | src.hi());
        ib.push(0x0F);
        ib.push(0x40 + cc as u8);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    // -- Bit manipulation (require BMI/POPCNT support) --

    /// popcnt r64, r64
    pub fn popcnt64(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0xF3);
        ib.push(0x48 | (dst.hi() << 2) | src.hi());
        ib.push(0x0F);
        ib.push(0xB8);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// lzcnt r64, r64
    pub fn lzcnt64(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0xF3);
        ib.push(0x48 | (dst.hi() << 2) | src.hi());
        ib.push(0x0F);
        ib.push(0xBD);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// tzcnt r64, r64
    pub fn tzcnt64(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0xF3);
        ib.push(0x48 | (dst.hi() << 2) | src.hi());
        ib.push(0x0F);
        ib.push(0xBC);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// popcnt r32, r32 (result zero-extended to 64 bits)
    pub fn popcnt32(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0xF3);
        let rex = (dst.hi() << 2) | src.hi();
        if rex != 0 {
            ib.push(0x40 | rex);
        }
        ib.push(0x0F);
        ib.push(0xB8);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// lzcnt r32, r32 (result zero-extended to 64 bits)
    pub fn lzcnt32(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0xF3);
        let rex = (dst.hi() << 2) | src.hi();
        if rex != 0 {
            ib.push(0x40 | rex);
        }
        ib.push(0x0F);
        ib.push(0xBD);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// tzcnt r32, r32 (result zero-extended to 64 bits)
    pub fn tzcnt32(&mut self, dst: Reg, src: Reg) {
        let mut ib = InstBuf::new();
        ib.push(0xF3);
        let rex = (dst.hi() << 2) | src.hi();
        if rex != 0 {
            ib.push(0x40 | rex);
        }
        ib.push(0x0F);
        ib.push(0xBC);
        ib.push(0xC0 | (dst.lo() << 3) | src.lo());
        self.flush_instbuf(ib);
    }

    /// bswap r64
    pub fn bswap64(&mut self, dst: Reg) {
        self.emit3(0x48 | dst.hi(), 0x0F, 0xC8 + dst.lo());
    }

    // -- Stack --

    #[inline(always)]
    pub fn push(&mut self, reg: Reg) {
        self.rex_opt_b(reg);
        self.emit(0x50 + reg.lo());
    }

    #[inline(always)]
    pub fn pop(&mut self, reg: Reg) {
        self.rex_opt_b(reg);
        self.emit(0x58 + reg.lo());
    }

    // -- Branches and jumps --

    /// jmp to label — uses rel8 for backward jumps within ±127 bytes.
    #[inline(always)]
    pub fn jmp_label(&mut self, label: Label) {
        let bound = self.labels[label.0 as usize];
        if bound != LABEL_UNBOUND {
            let target = (bound - 1) as isize; // stored as offset+1
            // Backward jump — label already bound, try rel8.
            let rel = target - (self.write_pos as isize + 2);
            if rel >= i8::MIN as isize && rel <= i8::MAX as isize {
                self.emit(0xEB);
                self.emit(rel as u8);
                return;
            }
        }
        // Forward jump or out of rel8 range — use rel32
        self.emit(0xE9);
        self.emit_label_fixup(label);
    }

    /// jcc to label — uses rel8 for backward jumps within ±127 bytes.
    #[inline(always)]
    pub fn jcc_label(&mut self, cc: Cc, label: Label) {
        let bound = self.labels[label.0 as usize];
        if bound != LABEL_UNBOUND {
            let target = (bound - 1) as isize; // stored as offset+1
            // Backward jump — label already bound, try rel8.
            let rel = target - (self.write_pos as isize + 2);
            if rel >= i8::MIN as isize && rel <= i8::MAX as isize {
                self.emit(0x70 + cc as u8);
                self.emit(rel as u8);
                return;
            }
        }
        // Forward jump or out of rel8 range — use rel32
        self.emit(0x0F);
        self.emit(0x80 + cc as u8);
        self.emit_label_fixup(label);
    }

    /// jmp r64 (indirect)
    pub fn jmp_reg(&mut self, reg: Reg) {
        self.rex_opt_b(reg);
        self.emit(0xFF);
        self.emit(0xE0 | reg.lo()); // /4
    }

    /// ret
    pub fn ret(&mut self) {
        self.emit(0xC3);
    }

    // -- LEA --

    /// lea r64, [base + disp]
    pub fn lea(&mut self, dst: Reg, base: Reg, disp: i32) {
        let mut ib = InstBuf::new();
        ib.push(0x48 | (dst.hi() << 2) | base.hi());
        ib.push(0x8D);
        Self::modrm_disp_ib(&mut ib, dst.lo(), base, disp);
        self.flush_instbuf(ib);
    }

    /// lea r32, [base + disp] — 32-bit result, zero-extends to 64-bit.
    #[inline(always)]
    pub fn lea_32(&mut self, dst: Reg, base: Reg, disp: i32) {
        let mut ib = InstBuf::new();
        let r = dst.hi();
        let b = base.hi();
        if r != 0 || b != 0 {
            ib.push(0x40 | (r << 2) | b);
        }
        ib.push(0x8D);
        Self::modrm_disp_ib(&mut ib, dst.lo(), base, disp);
        self.flush_instbuf(ib);
    }

    /// lea r32, [base32 + index32 * (1 << scale_log2)]
    /// scale_log2: 0=*1, 1=*2, 2=*4, 3=*8
    #[inline(always)]
    pub fn lea_sib_scaled_32(&mut self, dst: Reg, base: Reg, index: Reg, scale_log2: u8) {
        debug_assert!(scale_log2 <= 3);
        let mut ib = InstBuf::new();
        let rex = 0x40 | (dst.hi() << 2) | (index.hi() << 1) | base.hi();
        if rex != 0x40 {
            ib.push(rex);
        }
        ib.push(0x8D);
        let scale_bits = scale_log2 << 6;
        if base.lo() == 5 {
            ib.push(0x44 | (dst.lo() << 3));
            ib.push(scale_bits | (index.lo() << 3) | base.lo());
            ib.push(0x00);
        } else {
            ib.push((dst.lo() << 3) | 0x04);
            ib.push(scale_bits | (index.lo() << 3) | base.lo());
        }
        self.flush_instbuf(ib);
    }

    // -- Misc --

    /// nop
    pub fn nop(&mut self) {
        self.emit(0x90);
    }

    // === Finalization ===

    /// Get the resolved native offset for a label (only valid after bind_label).
    pub fn label_offset(&self, label: Label) -> Option<usize> {
        let off = self.labels[label.0 as usize];
        if off == LABEL_UNBOUND {
            None
        } else {
            Some((off - 1) as usize)
        }
    }

    /// Current number of pending fixups. Capture this before emitting a
    /// label-referencing instruction so the caller can later `redirect_fixup`
    /// the entry that the emission appends. Forward references push exactly
    /// one fixup; backward references push none (resolved immediately).
    #[inline]
    pub fn fixups_len(&self) -> usize {
        self.fixups.len()
    }

    /// Rewrite the target label of an existing fixup. Used by RV streaming
    /// compile to redirect forward branches whose target is determined to
    /// be invalid only after the streaming pass completes.
    pub fn redirect_fixup(&mut self, idx: usize, new_label: Label) {
        self.fixups[idx].label = new_label;
    }

    /// Sync Vec length with the write cursor. Call before accessing `self.code` directly.
    #[cfg(test)]
    pub fn sync_len(&mut self) {
        let code = &mut self.code_buf;
        // SAFETY: write_pos <= code.capacity() (maintained by emission guards).
        unsafe {
            code.set_len(self.write_pos);
        }
    }

    /// Resolve all label fixups in-place.
    fn resolve_fixups(&mut self) {
        for fixup in &self.fixups {
            let stored = self.labels[fixup.label.0 as usize];
            // All labels must be bound by finalization time.
            assert!(stored != LABEL_UNBOUND, "unbound label {:?}", fixup.label);
            let target = stored - 1; // stored as offset+1
            let rel = (target as i64) - (fixup.offset as i64 + 4);
            let rel32 = rel as i32;
            // SAFETY: fixup.offset + 4 <= write_pos (fixup was recorded during emission).
            unsafe {
                core::ptr::copy_nonoverlapping(
                    rel32.to_le_bytes().as_ptr(),
                    self.buf.add(fixup.offset),
                    4,
                );
            }
        }
    }

    /// Resolve fixups and return the code as a `Vec<u8>`.
    pub fn finalize(&mut self) -> Vec<u8> {
        self.resolve_fixups();
        let code = &mut self.code_buf;
        // SAFETY: write_pos <= code.capacity().
        unsafe {
            code.set_len(self.write_pos);
        }
        core::mem::take(code)
    }

    /// Get a slice of the written code bytes (for tests). Syncs Vec len first.
    #[cfg(test)]
    pub fn code_bytes(&mut self) -> &[u8] {
        self.sync_len();
        let v = &self.code_buf;
        v.as_slice()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_mov_ri64_zero() {
        let mut asm = Assembler::new();
        asm.mov_ri64(Reg::RAX, 0);
        // xor eax, eax → 0x31 0xC0
        assert_eq!(asm.code_bytes(), &[0x31, 0xC0]);
    }

    #[test]
    fn test_mov_ri64_small() {
        let mut asm = Assembler::new();
        asm.mov_ri64(Reg::RAX, 42);
        // mov eax, 42 → 0xB8, 0x2A, 0x00, 0x00, 0x00
        assert_eq!(asm.code_bytes(), &[0xB8, 0x2A, 0x00, 0x00, 0x00]);
    }

    #[test]
    fn test_label_resolution() {
        let mut asm = Assembler::new();
        let lbl = asm.new_label();
        asm.jmp_label(lbl); // 5 bytes: E9 + 4-byte rel32
        asm.nop(); // 1 byte at offset 5
        asm.bind_label(lbl); // label at offset 6
        let code = asm.finalize();
        // rel32 = 6 - (0 + 4 + 1) = 6 - 5 = 1
        // Wait: fixup offset is 1 (after E9), target is 6
        // rel = 6 - (1 + 4) = 1
        assert_eq!(code[0], 0xE9);
        let rel = i32::from_le_bytes([code[1], code[2], code[3], code[4]]);
        assert_eq!(rel, 1); // skip over the nop
    }

    /// Baseless `mov r32, [rdx]`:
    /// REX (0x40) + 0x8B + ModRM (mod=00, reg=000=eax, r/m=010=rdx) = 3 bytes.
    /// Codegen relies on this being the 3-byte form (vs the 4-byte SIB).
    #[test]
    fn test_mov_load32_at_rdx() {
        let mut asm = Assembler::new();
        asm.mov_load32_at_index(Reg::RAX, Reg::RDX);
        assert_eq!(asm.code_bytes(), &[0x8B, 0x02]);
    }

    /// Same shape but with R8 as dst (REX.R=1) and R12 as idx
    /// (lo=4, needs SIB recovery to encode bare `[r12]`).
    #[test]
    fn test_mov_load64_at_r12_into_r8() {
        let mut asm = Assembler::new();
        asm.mov_load64_at_index(Reg::R8, Reg::R12);
        // REX.W=1 REX.R=1 REX.B=1 → 0x4D
        // opcode 0x8B
        // ModRM mod=00 reg=000 r/m=100 → 0x04
        // SIB scale=00 index=100=none base=100 → 0x24
        assert_eq!(asm.code_bytes(), &[0x4D, 0x8B, 0x04, 0x24]);
    }

    /// `mov eax, [rip+disp32]` reaches `target_va`. With jit_va_base=0
    /// and write_pos=0 after emit, post-instruction RIP is the 7th byte,
    /// so disp32 = target_va - 7.
    #[test]
    fn test_mov_load32_rip_rel() {
        let mut asm = Assembler::new();
        asm.set_jit_va_base(0x1_0000_0000); // 4 GiB
        asm.mov_load32_rip_rel(Reg::RAX, 0x1_0000_0040); // 64 bytes above base
        // 8B 05 <disp32>  — disp = 0x40 - (0 + 6) = 0x3A
        // Wait: instruction is 6 bytes (no REX needed). post_inst_rip = base + 6.
        // disp = 0x1_0000_0040 - (0x1_0000_0000 + 6) = 0x3A.
        assert_eq!(asm.code_bytes(), &[0x8B, 0x05, 0x3A, 0x00, 0x00, 0x00]);
    }

    /// `mov [rip+disp32], r15` with REX.W|REX.R = 0x4C.
    #[test]
    fn test_mov_store64_rip_rel_r15() {
        let mut asm = Assembler::new();
        asm.set_jit_va_base(0x1_0000_0000);
        asm.mov_store64_rip_rel(0x1_0000_0080, Reg::R15);
        // 4C 89 3D <disp32> — 7-byte instruction.
        // disp = 0x80 - (0 + 7) = 0x79.
        assert_eq!(
            asm.code_bytes(),
            &[0x4C, 0x89, 0x3D, 0x79, 0x00, 0x00, 0x00]
        );
    }

    /// `mov [rdx], eax` (32-bit store) — baseless form, 2 bytes.
    #[test]
    fn test_mov_store32_at_rdx() {
        let mut asm = Assembler::new();
        asm.mov_store32_at_index(Reg::RDX, Reg::RAX);
        assert_eq!(asm.code_bytes(), &[0x89, 0x02]);
    }

    /// `mov [rdx], r11d` — REX.R for r11 source.
    #[test]
    fn test_mov_store32_at_rdx_from_r11() {
        let mut asm = Assembler::new();
        asm.mov_store32_at_index(Reg::RDX, Reg::R11);
        assert_eq!(asm.code_bytes(), &[0x44, 0x89, 0x1A]);
    }

    /// `mov byte [rdx], sil` — src ∈ {SPL,BPL,SIL,DIL} requires a
    /// REX prefix to disambiguate from AH/CH/DH/BH. The bare `88 32`
    /// would decode as `mov [rdx], DH` which is silently wrong.
    #[test]
    fn test_mov_store8_at_rdx_from_sil() {
        let mut asm = Assembler::new();
        asm.mov_store8_at_index(Reg::RDX, Reg::RSI);
        assert_eq!(asm.code_bytes(), &[0x40, 0x88, 0x32]);
    }

    #[test]
    fn test_mov_store8_at_rdx_from_dil() {
        let mut asm = Assembler::new();
        asm.mov_store8_at_index(Reg::RDX, Reg::RDI);
        assert_eq!(asm.code_bytes(), &[0x40, 0x88, 0x3A]);
    }

    #[test]
    fn test_mov_store8_at_rdx_from_bpl() {
        let mut asm = Assembler::new();
        asm.mov_store8_at_index(Reg::RDX, Reg::RBP);
        assert_eq!(asm.code_bytes(), &[0x40, 0x88, 0x2A]);
    }

    #[test]
    fn test_mov_store8_at_rdx_from_spl() {
        let mut asm = Assembler::new();
        asm.mov_store8_at_index(Reg::RDX, Reg::RSP);
        assert_eq!(asm.code_bytes(), &[0x40, 0x88, 0x22]);
    }

    /// `mov byte [rdx], al` — REX still NOT emitted for AL/CL/DL/BL.
    #[test]
    fn test_mov_store8_at_rdx_from_al_no_rex() {
        let mut asm = Assembler::new();
        asm.mov_store8_at_index(Reg::RDX, Reg::RAX);
        assert_eq!(asm.code_bytes(), &[0x88, 0x02]);
    }

    /// `mov dword [rip+disp32], imm32` — RIP-relative store-imm form.
    /// 10-byte instruction: C7 05 <disp32> <imm32>. The CPU resolves
    /// RIP-relative against the NEXT-instruction RIP, which is past
    /// BOTH disp32 and the trailing imm32 — so disp = target - 10.
    #[test]
    fn test_mov_store32_rip_rel_imm() {
        let mut asm = Assembler::new();
        asm.set_jit_va_base(0x1_0000_0000);
        asm.mov_store32_rip_rel_imm(0x1_0000_0100, 0x123);
        // disp = 0x100 - (0 + 10) = 0xF6.
        assert_eq!(
            asm.code_bytes(),
            &[0xC7, 0x05, 0xF6, 0x00, 0x00, 0x00, 0x23, 0x01, 0x00, 0x00]
        );
    }

    /// `add rax, [rip+disp32]` — used by the dispatch sequence to
    /// fold code_base into native_addr.
    #[test]
    fn test_add_r64_mem_rip_rel() {
        let mut asm = Assembler::new();
        asm.set_jit_va_base(0x1_0000_0000);
        asm.add_r64_mem_rip_rel(Reg::RAX, 0x1_0000_0020);
        // 48 03 05 <disp32> — 7-byte instruction.
        // disp = 0x20 - (0 + 7) = 0x19.
        assert_eq!(
            asm.code_bytes(),
            &[0x48, 0x03, 0x05, 0x19, 0x00, 0x00, 0x00]
        );
    }

    /// `sub r15, imm32` patchable form — must be exactly 7 bytes with
    /// the imm32 at bytes 3..7 (codegen patches via `offset() - 4`).
    #[test]
    fn test_sub_r64_imm32_patchable_r15() {
        let mut asm = Assembler::new();
        asm.sub_r64_imm32_patchable(Reg::R15, 0xCAFE_F00D_u32 as i32);
        let bytes = asm.code_bytes();
        assert_eq!(bytes.len(), 7, "patchable sub must be 7 bytes");
        // REX.W=1 REX.B=1 → 0x49
        // opcode 0x81
        // ModRM mod=11 reg=5 r/m=7 → 0xEF
        // imm32 at bytes 3..7 in LE
        assert_eq!(&bytes[0..3], &[0x49, 0x81, 0xEF]);
        assert_eq!(&bytes[3..7], &[0x0D, 0xF0, 0xFE, 0xCA]);
    }

    #[test]
    fn test_push_pop_r15() {
        let mut asm = Assembler::new();
        asm.push(Reg::R15);
        asm.pop(Reg::R15);
        // push r15: 41 57, pop r15: 41 5F
        assert_eq!(asm.code_bytes(), &[0x41, 0x57, 0x41, 0x5F]);
    }
}