vatrog-vm-introspection-engine/include/insndec.h

/* insndec.h - readable per-instruction operand decode (generic, ABI-agnostic).
 *
 * Handler layer: a RICH decode of ONE x86-64 instruction into concrete operands
 * (concrete registers as a backend-neutral stable id, memory base/index/scale/
 * disp, immediate VALUES, sizes), per-operand read/write, AND the full aggregate
 * register read/write sets INCLUDING IMPLICIT operands/effects (rsp on push/pop,
 * rdx:rax on mul/div, FLAGS on cmp/jcc, ...). This is the "perception" primitive
 * the register def-use analysis (codetrace.h) is built on.
 *
 * Relationship to semsig.h (shared backend, DIFFERENT purpose). semsig_hash
 * folds a NORMALIZED, LOSSY token stream: it erases register identity (keeps only
 * a class), erases immediate/displacement values, and carries no read/write - by
 * design, so the hash is register-/value-/reorder-stable. That makes semsig
 * irreversible: you cannot trace registers through it. insn_decode is the
 * opposite: it KEEPS register identity, values, and read/write, so an analysis
 * can reason about WHICH physical register an instruction touches. Both ride the
 * SAME optional external disassembler (Zydis/Capstone) behind the same private
 * seam, but insn_decode is NOT a hash and NOT lossy.
 *
 * Operand decode is NOT in the light decoder (x86dec.h is deliberately pure and
 * length-only) and NOT a second hand-rolled operand decoder: it is supplied by
 * the optional backend, so this primitive is FEATURE-GATED exactly like semsig.
 * With no backend the symbol still exists (ABI stable) but returns 0, and
 * VMIE_HAVE_DISASM is 0.
 *
 * ABI-AGNOSTIC. insn_decode reports ONLY what an instruction does
 * ARCHITECTURALLY. In particular a `call` writes rsp and rip and (for an
 * indirect call) reads its target register/memory - that is ALL. It does NOT
 * mark caller-saved (volatile) registers as written: that is a property of the
 * CALLED function / the calling convention, not of the instruction. The Win64
 * volatile clobber lives in the def-use layer (codetrace.h/.c) behind an explicit
 * trace_abi, never here.
 */
#ifndef VMIE_INSNDEC_H
#define VMIE_INSNDEC_H
#include <stdint.h>
#include <stddef.h>

/* Compile-time feature availability, shared with semsig (one backend, one gate).
 * Pushed from CMake as a PUBLIC compile definition so consumers see the SAME
 * value the library was built with:
 *   VMIE_HAVE_DISASM == 1  built with a disassembler backend; insn_decode is
 *                          live, insndec_backend_name() names the backend.
 *   VMIE_HAVE_DISASM == 0  built without a backend (the default); insn_decode
 *                          always returns 0 and insndec_backend_name() == "none".
 * The default here keeps the header valid when compiled outside the project. */
#ifndef VMIE_HAVE_DISASM
#define VMIE_HAVE_DISASM 0
#endif

/* Canonical full-register root id. al/ah/ax/eax/rax all map to REGID_RAX;
 * xmm0/ymm0/zmm0 -> REGID_VEC0; etc. Identity is KEPT here (unlike sem_insn,
 * which erases it) so def-use can reason about WHICH physical register. The
 * concrete backend register enum is NEVER carried across the seam - the backend
 * TU normalizes into THIS stable id, exactly as it does for mnemonic classes. */
typedef enum {
    REGID_NONE = 0,
    /* 16 GPR roots; al/ax/eax/rax -> REGID_RAX, etc. */
    REGID_RAX, REGID_RCX, REGID_RDX, REGID_RBX,
    REGID_RSP, REGID_RBP, REGID_RSI, REGID_RDI,
    REGID_R8,  REGID_R9,  REGID_R10, REGID_R11,
    REGID_R12, REGID_R13, REGID_R14, REGID_R15,
    /* 32 vector roots (zmm0..zmm31; xmm/ymm fold into the zmm root). */
    REGID_VEC0,
    REGID_VEC31 = REGID_VEC0 + 31,
    /* AVX-512 mask k0..k7. */
    REGID_K0,
    REGID_K7 = REGID_K0 + 7,
    /* FLAGS as ONE pseudo-register root (NOT per-bit CF/ZF/...): enough for
     * def-use "cmp writes flags, jcc/cmov/setcc read flags"; per-bit is not v1. */
    REGID_FLAGS,
    /* rip / segment / other roots kept coarse (def-use rarely tracks them). */
    REGID_OTHER,
    REGID_COUNT
} reg_id;

/* Per-operand / per-register access, normalized (raw backend enums never leak). */
typedef enum {
    ACC_NONE = 0,
    ACC_READ,        /* read only                                   */
    ACC_WRITE,       /* write only                                  */
    ACC_READWRITE,   /* read-modify-write (add, sub, ...)           */
    ACC_COND_READ,   /* conditionally read  (cmovcc src, ...)       */
    ACC_COND_WRITE   /* conditionally written (cmovcc dst, setcc)   */
} reg_access;

/* Explicit-operand kind (the readable, perception side of the decode). */
typedef enum {
    OPND_NONE = 0,
    OPND_REG,        /* register operand                            */
    OPND_MEM,        /* memory operand (base/index/scale/disp)      */
    OPND_IMM         /* immediate operand (value kept)              */
} operand_kind;

/* One register reference, used both for explicit register operands and for the
 * aggregate implicit-or-explicit read/write sets. Sub-register overlap is modeled
 * EXPLICITLY and backend-neutrally:
 *   root              - canonical full-register root (al/ax/eax/rax -> REGID_RAX)
 *   width_log2        - ACCESS width in log2 bytes (0=1B .. 4=16B), as
 *                       width_log2_of already computes for semsig
 *   writes_zero_extend- on a WRITE, 1 if the upper part of the root is zeroed
 *                       (a 32-bit GPR write, e.g. `eax`, zeroes the upper 32 bits
 *                       of `rax`; a VEX/EVEX vector write zeroes the upper YMM/ZMM)
 *                       and 0 if it is a PARTIAL write that preserves the upper
 *                       part (8/16-bit GPR write, legacy-SSE vector write). This
 *                       ONE flag is what def-use needs to decide overlap without
 *                       carrying a raw bit mask.
 *   access            - ACC_* for this reference */
typedef struct {
    uint8_t root;             /* reg_id                                         */
    uint8_t width_log2;       /* access width, log2 bytes (0..4+)               */
    uint8_t writes_zero_extend;
    uint8_t access;           /* reg_access                                     */
} reg_ref;

/* One explicit operand (readable / perception view). */
typedef struct {
    uint8_t kind;             /* operand_kind                                   */
    uint8_t access;           /* reg_access for the operand as a whole          */
    uint8_t width_log2;       /* access width, log2 bytes                       */
    uint8_t mem_scale;        /* MEM: index scale (1/2/4/8), 0 if no index      */
    uint8_t mem_is_riprel;    /* MEM: 1 if RIP-relative                         */
    uint8_t reg_root;         /* REG: reg_id of the operand register            */
    uint8_t reg_zero_extend;  /* REG: writes_zero_extend for a write operand    */
    uint8_t imm_width_log2;   /* IMM: value width, log2 bytes                   */
    uint8_t mem_base;         /* MEM: reg_id of the base register (read)        */
    uint8_t mem_index;        /* MEM: reg_id of the index register (read)       */
    int64_t mem_disp;         /* MEM: displacement value (signed)               */
    int64_t imm_value;        /* IMM: immediate value (sign-extended)           */
} operand;

/* Mnemonic-class group, backend-neutral. Mirrors the semsig SEM_MN_* groups so a
 * consumer can switch on the instruction kind without backend enums; def-use uses
 * only INSN_CALL (call sites) but the rest aids readability of the decode. */
typedef enum {
    INSN_OTHER = 0,
    INSN_MOV, INSN_ARITH, INSN_LOGIC, INSN_CMP, INSN_TEST,
    INSN_BRANCH,             /* conditional jcc / setcc / cmovcc               */
    INSN_JMP,                /* unconditional jmp                              */
    INSN_CALL, INSN_RET, INSN_PUSH, INSN_POP, INSN_LEA, INSN_NOP,
    INSN_FLOAT, INSN_VEC
} insn_class;

/* Caps on the rich decode. Four explicit operands cover x86-64; eight register
 * refs per side cover the widest implicit sets (e.g. mul touching rax/rdx plus an
 * explicit source) with margin. Anything beyond is truncated (count still grows
 * past the cap so a caller can detect it), which never happens for real code. */
#define INSN_MAX_OPERANDS 4
#define INSN_MAX_REGREFS  8

/* A fully decoded instruction. Two complementary representations:
 *   op[]    - explicit operands, in instruction operand order (readability);
 *   reads[] / writes[] - aggregate register sets over canonical roots, UNIONING
 *             explicit AND implicit effects (this is what def-use consumes). A
 *             memory operand's base/index registers appear in reads[] as address
 *             reads; FLAGS appears as REGID_FLAGS; rsp/rdx:rax appear via the
 *             backend's implicit operands. */
typedef struct {
    uint8_t  length;          /* full instruction length in bytes (>= 1)        */
    uint8_t  mnemonic_class;  /* insn_class                                     */
    uint8_t  is_control_flow; /* 1 if branch/call/ret                           */
    uint8_t  noperands;       /* explicit operands in op[] (<= INSN_MAX_OPERANDS)*/
    uint8_t  nreads;          /* register refs in reads[]                       */
    uint8_t  nwrites;         /* register refs in writes[]                      */
    operand  op[INSN_MAX_OPERANDS];
    reg_ref  reads[INSN_MAX_REGREFS];
    reg_ref  writes[INSN_MAX_REGREFS];
} insn_decoded;

/* Decode ONE instruction at view[off .. ]. `view` is a flat byte span and `off`
 * is the byte offset of the instruction within it; at most (view_size - off)
 * bytes are available to the backend. Returns 1 on success (fills *out), 0 on
 * undecodable / not enough bytes / a build with no backend (VMIE_HAVE_DISASM==0).
 * out->length == 0 is also treated as a desync by callers. PURE: only the byte
 * span and the decoder, no I/O, no allocation, reentrant.
 *
 * `view` / `view_size` are a flat buffer (not mem_view_t) so this primitive
 * stays dependency-light; the caller owns the buffer.
 *
 * Example - read the registers an instruction writes:
 *   insn_decoded d;
 *   if (insn_decode(code, n, 0, &d))
 *       for (int i = 0; i < d.nwrites; i++)
 *           printf("writes root %u (zx=%u)\n",
 *                  d.writes[i].root, d.writes[i].writes_zero_extend); */
int insn_decode(const uint8_t* view, size_t view_size, size_t off,
                insn_decoded* out);

/* Stable identity of the compiled disassembler backend ("zydis"/"capstone", or
 * "none" without a backend). Shares the backend with semsig; this name is the
 * same backend. Static, never-NULL. */
const char* insndec_backend_name(void);

#endif /* VMIE_INSNDEC_H */