vatrog-vm-introspection-engine/src/handlers/semsig_backend_capstone.c

/* semsig_backend_capstone.c - Capstone implementation of the operand-decode seam.
 *
 * The ONE TU that includes Capstone headers and links its library. It implements
 * the semsig_backend.h contract: decode one x86-64 instruction into a sem_insn
 * (length, backend-neutral mnemonic class, normalized operands), and report the
 * backend name. Built ONLY when VMIE_DISASM=capstone (see CMakeLists.txt).
 *
 * Stateless: a Capstone handle is opened/closed on the call stack per decode, no
 * global mutable state - reentrant. Capstone allocates internally for cs_disasm;
 * the buffer is freed before return so there is no leak across the seam.
 */
#include <stdint.h>
#include <stddef.h>
#include <capstone/capstone.h>
#include "semsig_backend.h"

/* Map a Capstone x86 instruction id to a backend-neutral SEM_MN_* class. STATIC
 * table, not a switch forest; unlisted ids fall through to SEM_MN_OTHER (where
 * the core folds the raw opcode). Conditional branches (Jcc) are a family folded
 * by Capstone group, handled before the table lookup. */
typedef struct { unsigned int id; uint16_t cls; } mn_map;

static const mn_map MN_TABLE[] = {
    /* moves */
    { X86_INS_MOV,    SEM_MN_MOV },
    { X86_INS_MOVZX,  SEM_MN_MOV },
    { X86_INS_MOVSX,  SEM_MN_MOV },
    { X86_INS_MOVSXD, SEM_MN_MOV },
    { X86_INS_MOVAPS, SEM_MN_MOV },
    { X86_INS_MOVAPD, SEM_MN_MOV },
    { X86_INS_MOVDQA, SEM_MN_MOV },
    { X86_INS_MOVDQU, SEM_MN_MOV },
    { X86_INS_MOVD,   SEM_MN_MOV },
    { X86_INS_MOVQ,   SEM_MN_MOV },
    { X86_INS_XCHG,   SEM_MN_MOV },
    { X86_INS_LEA,    SEM_MN_LEA },
    /* arithmetic */
    { X86_INS_ADD,    SEM_MN_ARITH },
    { X86_INS_ADC,    SEM_MN_ARITH },
    { X86_INS_SUB,    SEM_MN_ARITH },
    { X86_INS_SBB,    SEM_MN_ARITH },
    { X86_INS_INC,    SEM_MN_ARITH },
    { X86_INS_DEC,    SEM_MN_ARITH },
    { X86_INS_NEG,    SEM_MN_ARITH },
    { X86_INS_MUL,    SEM_MN_ARITH },
    { X86_INS_IMUL,   SEM_MN_ARITH },
    { X86_INS_DIV,    SEM_MN_ARITH },
    { X86_INS_IDIV,   SEM_MN_ARITH },
    /* logic / shifts */
    { X86_INS_AND,    SEM_MN_LOGIC },
    { X86_INS_OR,     SEM_MN_LOGIC },
    { X86_INS_XOR,    SEM_MN_LOGIC },
    { X86_INS_NOT,    SEM_MN_LOGIC },
    { X86_INS_SHL,    SEM_MN_LOGIC },
    { X86_INS_SHR,    SEM_MN_LOGIC },
    { X86_INS_SAR,    SEM_MN_LOGIC },
    { X86_INS_ROL,    SEM_MN_LOGIC },
    { X86_INS_ROR,    SEM_MN_LOGIC },
    /* compare / test */
    { X86_INS_CMP,    SEM_MN_CMP },
    { X86_INS_TEST,   SEM_MN_TEST },
    /* control flow */
    { X86_INS_JMP,    SEM_MN_JMP },
    { X86_INS_CALL,   SEM_MN_CALL },
    { X86_INS_RET,    SEM_MN_RET },
    { X86_INS_RETF,   SEM_MN_RET },
    { X86_INS_RETFQ,  SEM_MN_RET },
    { X86_INS_PUSH,   SEM_MN_PUSH },
    { X86_INS_POP,    SEM_MN_POP },
    { X86_INS_NOP,    SEM_MN_NOP },
};

/* True if `g` is among the instruction's Capstone groups. */
static int has_group(const cs_insn* in, uint8_t g) {
    const cs_detail* d = in->detail;
    if (!d) { return 0; }
    for (uint8_t i = 0; i < d->groups_count; i++) {
        if (d->groups[i] == g) { return 1; }
    }
    return 0;
}

static uint16_t classify_mnemonic(const cs_insn* in) {
    if (has_group(in, X86_GRP_CALL)) { return SEM_MN_CALL; }
    if (has_group(in, X86_GRP_RET))  { return SEM_MN_RET; }
    if (has_group(in, X86_GRP_JUMP)) {
        /* unconditional jmp vs conditional jcc: JMP id is unconditional. */
        return (in->id == X86_INS_JMP) ? SEM_MN_JMP : SEM_MN_BRANCH;
    }
    for (size_t i = 0; i < sizeof MN_TABLE / sizeof MN_TABLE[0]; i++) {
        if (MN_TABLE[i].id == in->id) { return MN_TABLE[i].cls; }
    }
    if (has_group(in, X86_GRP_FPU)) { return SEM_MN_FLOAT; }
    if (has_group(in, X86_GRP_SSE) || has_group(in, X86_GRP_SSE1) ||
        has_group(in, X86_GRP_SSE2)) { return SEM_MN_VEC; }
    return SEM_MN_OTHER;
}

/* Map a Capstone x86 register id to a canonical SEM_RC_* class. Capstone has no
 * single "register class" accessor; classify by the documented id ranges. */
static uint8_t reg_class_of(unsigned int reg) {
    if (reg == X86_REG_INVALID) { return SEM_RC_NONE; }
    if ((reg >= X86_REG_AH && reg <= X86_REG_BH) ||
        (reg >= X86_REG_AL && reg <= X86_REG_R15D) ||
        (reg >= X86_REG_RAX && reg <= X86_REG_RBP) ||
        (reg >= X86_REG_RSP && reg <= X86_REG_RDI) ||
        (reg >= X86_REG_R8 && reg <= X86_REG_R15)) { return SEM_RC_GPR; }
    if (reg >= X86_REG_XMM0 && reg <= X86_REG_ZMM31) { return SEM_RC_XMM; }
    if (reg >= X86_REG_K0 && reg <= X86_REG_K7) { return SEM_RC_MASK; }
    if (reg >= X86_REG_CS && reg <= X86_REG_SS) { return SEM_RC_SEG; }
    if (reg >= X86_REG_EFLAGS && reg <= X86_REG_RIP) { return SEM_RC_FLAGS; }
    return SEM_RC_OTHER;
}

static uint8_t width_log2_of(uint8_t size_bytes) {
    uint8_t l = 0;
    uint8_t v = size_bytes;
    while (v > 1u && l < 4u) { v >>= 1; l++; }
    return l;
}

int semsig_backend_decode(const uint8_t* code, size_t avail, sem_insn* out) {
    if (!code || !out || avail == 0) { return 0; }

    csh h;
    if (cs_open(CS_ARCH_X86, CS_MODE_64, &h) != CS_ERR_OK) { return 0; }
    cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);

    cs_insn* insn = NULL;
    const size_t n = cs_disasm(h, code, avail, 0, 1, &insn);
    if (n < 1 || !insn) {
        if (insn) { cs_free(insn, n); }
        cs_close(&h);
        return 0;
    }

    for (size_t i = 0; i < sizeof *out; i++) { ((uint8_t*)out)[i] = 0; }
    out->length         = (uint8_t)insn->size;
    out->mnemonic_class = classify_mnemonic(insn);
    out->is_control_flow =
        (has_group(insn, X86_GRP_JUMP) || has_group(insn, X86_GRP_CALL) ||
         has_group(insn, X86_GRP_RET)) ? 1u : 0u;
    out->raw_opcode = (out->mnemonic_class == SEM_MN_OTHER && insn->detail)
                      ? insn->detail->x86.opcode[0] : 0u;

    uint8_t no = 0;
    if (insn->detail) {
        const cs_x86* x = &insn->detail->x86;
        for (uint8_t i = 0; i < x->op_count && no < 4; i++) {
            const cs_x86_op* o = &x->operands[i];
            switch (o->type) {
                case X86_OP_REG:
                    out->op[no].kind       = SEMOP_REG;
                    out->op[no].reg_class  = reg_class_of(o->reg);
                    out->op[no].width_log2 = width_log2_of(o->size);
                    no++;
                    break;
                case X86_OP_MEM:
                    out->op[no].kind          = SEMOP_MEM;
                    out->op[no].width_log2    = width_log2_of(o->size);
                    out->op[no].mem_has_base  =
                        (o->mem.base != X86_REG_INVALID &&
                         o->mem.base != X86_REG_RIP) ? 1u : 0u;
                    out->op[no].mem_has_index =
                        (o->mem.index != X86_REG_INVALID) ? 1u : 0u;
                    out->op[no].mem_is_riprel =
                        (o->mem.base == X86_REG_RIP) ? 1u : 0u;
                    no++;
                    break;
                case X86_OP_IMM:
                    out->op[no].kind       = SEMOP_IMM;
                    out->op[no].width_log2 = width_log2_of(o->size);
                    no++;
                    break;
                default:
                    break;
            }
        }
    }
    out->noperands = no;

    cs_free(insn, n);
    cs_close(&h);
    return 1;
}

/* ---- rich decode (insn_decode_full): the lossless projection ------------- */

/* Map a Capstone x86 register id to a canonical reg_id ROOT (al/ax/eax/rax ->
 * REGID_RAX, xmm/ymm/zmm -> a REGID_VEC* root, k0..k7 -> REGID_K*). Capstone has
 * no "largest enclosing" helper, so fold by the documented id ranges (the same
 * approach reg_class_of already uses). The raw x86_reg never leaves this TU. */
static uint8_t reg_root_of(unsigned int reg) {
    switch (reg) {
        case X86_REG_AL: case X86_REG_AH: case X86_REG_AX:
        case X86_REG_EAX: case X86_REG_RAX: return REGID_RAX;
        case X86_REG_CL: case X86_REG_CH: case X86_REG_CX:
        case X86_REG_ECX: case X86_REG_RCX: return REGID_RCX;
        case X86_REG_DL: case X86_REG_DH: case X86_REG_DX:
        case X86_REG_EDX: case X86_REG_RDX: return REGID_RDX;
        case X86_REG_BL: case X86_REG_BH: case X86_REG_BX:
        case X86_REG_EBX: case X86_REG_RBX: return REGID_RBX;
        case X86_REG_SPL: case X86_REG_SP: case X86_REG_ESP:
        case X86_REG_RSP: return REGID_RSP;
        case X86_REG_BPL: case X86_REG_BP: case X86_REG_EBP:
        case X86_REG_RBP: return REGID_RBP;
        case X86_REG_SIL: case X86_REG_SI: case X86_REG_ESI:
        case X86_REG_RSI: return REGID_RSI;
        case X86_REG_DIL: case X86_REG_DI: case X86_REG_EDI:
        case X86_REG_RDI: return REGID_RDI;
        case X86_REG_R8B: case X86_REG_R8W: case X86_REG_R8D:
        case X86_REG_R8: return REGID_R8;
        case X86_REG_R9B: case X86_REG_R9W: case X86_REG_R9D:
        case X86_REG_R9: return REGID_R9;
        case X86_REG_R10B: case X86_REG_R10W: case X86_REG_R10D:
        case X86_REG_R10: return REGID_R10;
        case X86_REG_R11B: case X86_REG_R11W: case X86_REG_R11D:
        case X86_REG_R11: return REGID_R11;
        case X86_REG_R12B: case X86_REG_R12W: case X86_REG_R12D:
        case X86_REG_R12: return REGID_R12;
        case X86_REG_R13B: case X86_REG_R13W: case X86_REG_R13D:
        case X86_REG_R13: return REGID_R13;
        case X86_REG_R14B: case X86_REG_R14W: case X86_REG_R14D:
        case X86_REG_R14: return REGID_R14;
        case X86_REG_R15B: case X86_REG_R15W: case X86_REG_R15D:
        case X86_REG_R15: return REGID_R15;
        default: break;
    }
    if (reg >= X86_REG_XMM0 && reg <= X86_REG_XMM31) {
        return (uint8_t)(REGID_VEC0 + (reg - X86_REG_XMM0));
    }
    if (reg >= X86_REG_YMM0 && reg <= X86_REG_YMM31) {
        return (uint8_t)(REGID_VEC0 + (reg - X86_REG_YMM0));
    }
    if (reg >= X86_REG_ZMM0 && reg <= X86_REG_ZMM31) {
        return (uint8_t)(REGID_VEC0 + (reg - X86_REG_ZMM0));
    }
    if (reg >= X86_REG_K0 && reg <= X86_REG_K7) {
        return (uint8_t)(REGID_K0 + (reg - X86_REG_K0));
    }
    if (reg == X86_REG_EFLAGS) { return REGID_FLAGS; }
    if (reg == X86_REG_INVALID) { return REGID_NONE; }
    return REGID_OTHER;
}

/* Approximate the ACCESS width of a register in log2 bytes from its sub-register
 * id (Capstone register ids encode the width); used when only an id is known
 * (cs_regs_access returns ids without sizes). Coarse but enough for overlap. */
static uint8_t reg_width_log2(unsigned int reg) {
    switch (reg) {
        case X86_REG_AL: case X86_REG_AH: case X86_REG_CL: case X86_REG_CH:
        case X86_REG_DL: case X86_REG_DH: case X86_REG_BL: case X86_REG_BH:
        case X86_REG_SPL: case X86_REG_BPL: case X86_REG_SIL: case X86_REG_DIL:
        case X86_REG_R8B: case X86_REG_R9B: case X86_REG_R10B:
        case X86_REG_R11B: case X86_REG_R12B: case X86_REG_R13B:
        case X86_REG_R14B: case X86_REG_R15B: return 0;
        case X86_REG_AX: case X86_REG_CX: case X86_REG_DX: case X86_REG_BX:
        case X86_REG_SP: case X86_REG_BP: case X86_REG_SI: case X86_REG_DI:
        case X86_REG_R8W: case X86_REG_R9W: case X86_REG_R10W:
        case X86_REG_R11W: case X86_REG_R12W: case X86_REG_R13W:
        case X86_REG_R14W: case X86_REG_R15W: return 1;
        case X86_REG_EAX: case X86_REG_ECX: case X86_REG_EDX: case X86_REG_EBX:
        case X86_REG_ESP: case X86_REG_EBP: case X86_REG_ESI: case X86_REG_EDI:
        case X86_REG_R8D: case X86_REG_R9D: case X86_REG_R10D:
        case X86_REG_R11D: case X86_REG_R12D: case X86_REG_R13D:
        case X86_REG_R14D: case X86_REG_R15D: return 2;
        default: break;
    }
    if (reg >= X86_REG_XMM0 && reg <= X86_REG_XMM31) { return 4; }
    return 3;   /* 64-bit GPR / vector-as-root / other: default qword */
}

/* Backend-neutral mnemonic CLASS for the rich decode (insn_class enum), by
 * translating the existing classify_mnemonic SEM_MN_* result. */
static uint8_t insn_class_of(const cs_insn* in) {
    switch (classify_mnemonic(in)) {
        case SEM_MN_MOV:    return INSN_MOV;
        case SEM_MN_ARITH:  return INSN_ARITH;
        case SEM_MN_LOGIC:  return INSN_LOGIC;
        case SEM_MN_CMP:    return INSN_CMP;
        case SEM_MN_TEST:   return INSN_TEST;
        case SEM_MN_BRANCH: return INSN_BRANCH;
        case SEM_MN_JMP:    return INSN_JMP;
        case SEM_MN_CALL:   return INSN_CALL;
        case SEM_MN_RET:    return INSN_RET;
        case SEM_MN_PUSH:   return INSN_PUSH;
        case SEM_MN_POP:    return INSN_POP;
        case SEM_MN_LEA:    return INSN_LEA;
        case SEM_MN_NOP:    return INSN_NOP;
        case SEM_MN_FLOAT:  return INSN_FLOAT;
        case SEM_MN_VEC:    return INSN_VEC;
        default:            return INSN_OTHER;
    }
}

/* Normalize a Capstone cs_ac_type access mask to a reg_access. */
static uint8_t access_of(uint8_t a) {
    const int r = (a & CS_AC_READ)  != 0;
    const int w = (a & CS_AC_WRITE) != 0;
    if (r && w) { return ACC_READWRITE; }
    if (r)      { return ACC_READ; }
    if (w)      { return ACC_WRITE; }
    return ACC_NONE;
}

/* writes_zero_extend rule (see insndec.h): 32-bit GPR write zero-extends; EVEX/
 * VEX vector write zero-extends the upper part; 8/16-bit and legacy-SSE are
 * partial. */
static uint8_t zx_of(uint8_t root, uint8_t width_log2, int vector_evex_vex) {
    if (root >= REGID_RAX && root <= REGID_R15) {
        return (width_log2 == 2) ? 1u : 0u;
    }
    if (root >= REGID_VEC0 && root <= REGID_VEC31) {
        return vector_evex_vex ? 1u : 0u;
    }
    return 0u;
}

static void regset_add(reg_ref* set, uint8_t* n, uint8_t root,
                       uint8_t width_log2, uint8_t zx, uint8_t access) {
    if (root == REGID_NONE) { return; }
    for (uint8_t i = 0; i < *n && i < INSN_MAX_REGREFS; i++) {
        if (set[i].root == root) {
            if (width_log2 > set[i].width_log2) {
                set[i].width_log2 = width_log2;
                set[i].writes_zero_extend = zx;
            }
            return;
        }
    }
    if (*n < INSN_MAX_REGREFS) {
        set[*n].root = root;
        set[*n].width_log2 = width_log2;
        set[*n].writes_zero_extend = zx;
        set[*n].access = access;
    }
    (*n)++;
}

int insn_decode_full(const uint8_t* code, size_t avail, insn_decoded* out) {
    if (!code || !out || avail == 0) { return 0; }

    csh h;
    if (cs_open(CS_ARCH_X86, CS_MODE_64, &h) != CS_ERR_OK) { return 0; }
    cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);

    cs_insn* insn = NULL;
    const size_t n = cs_disasm(h, code, avail, 0, 1, &insn);
    if (n < 1 || !insn) {
        if (insn) { cs_free(insn, n); }
        cs_close(&h);
        return 0;
    }

    for (size_t i = 0; i < sizeof *out; i++) { ((uint8_t*)out)[i] = 0; }
    out->length          = (uint8_t)insn->size;
    out->mnemonic_class  = insn_class_of(insn);
    out->is_control_flow =
        (has_group(insn, X86_GRP_JUMP) || has_group(insn, X86_GRP_CALL) ||
         has_group(insn, X86_GRP_RET)) ? 1u : 0u;

    if (insn->detail) {
        const cs_x86* x = &insn->detail->x86;
        /* A VEX/EVEX form zero-extends the upper part of a written vector root;
         * legacy-SSE preserves it. Detect by the VEX/EVEX leading opcode byte. */
        const int is_vex_evex =
            (x->opcode[0] == 0xC4 || x->opcode[0] == 0xC5 || x->opcode[0] == 0x62);

        /* Explicit operands (readable view): op[]. */
        uint8_t no = 0;
        for (uint8_t i = 0; i < x->op_count && no < INSN_MAX_OPERANDS; i++) {
            const cs_x86_op* o = &x->operands[i];
            const uint8_t acc = access_of(o->access);
            const uint8_t wl  = width_log2_of(o->size);
            switch (o->type) {
                case X86_OP_REG: {
                    const uint8_t root = reg_root_of(o->reg);
                    out->op[no].kind            = OPND_REG;
                    out->op[no].access          = acc;
                    out->op[no].width_log2      = wl;
                    out->op[no].reg_root        = root;
                    out->op[no].reg_zero_extend = zx_of(root, wl, is_vex_evex);
                    no++;
                    break;
                }
                case X86_OP_MEM:
                    out->op[no].kind          = OPND_MEM;
                    out->op[no].access        = acc;
                    out->op[no].width_log2    = wl;
                    out->op[no].mem_base      =
                        (o->mem.base == X86_REG_RIP) ? REGID_NONE
                                                     : reg_root_of(o->mem.base);
                    out->op[no].mem_index     = reg_root_of(o->mem.index);
                    out->op[no].mem_scale     = (uint8_t)o->mem.scale;
                    out->op[no].mem_is_riprel =
                        (o->mem.base == X86_REG_RIP) ? 1u : 0u;
                    out->op[no].mem_disp      = o->mem.disp;
                    no++;
                    break;
                case X86_OP_IMM:
                    out->op[no].kind           = OPND_IMM;
                    out->op[no].access         = ACC_READ;
                    out->op[no].imm_width_log2 = wl;
                    out->op[no].imm_value      = o->imm;
                    no++;
                    break;
                default:
                    break;
            }
        }
        out->noperands = no;

        /* Aggregate implicit-or-explicit register sets via cs_regs_access: the
         * FULL read/write sets including rsp/rdx:rax/EFLAGS. EFLAGS -> REGID_FLAGS;
         * vector writes carry zero-extend by form. */
        cs_regs rd, wr;
        uint8_t nrd = 0, nwr = 0;
        if (cs_regs_access(h, insn, rd, &nrd, wr, &nwr) == CS_ERR_OK) {
            for (uint8_t i = 0; i < nrd; i++) {
                const uint8_t root = reg_root_of(rd[i]);
                regset_add(out->reads, &out->nreads, root,
                           reg_width_log2(rd[i]), 0, ACC_READ);
            }
            for (uint8_t i = 0; i < nwr; i++) {
                const uint8_t root = reg_root_of(wr[i]);
                const uint8_t wl = reg_width_log2(wr[i]);
                regset_add(out->writes, &out->nwrites, root, wl,
                           zx_of(root, wl, is_vex_evex), ACC_WRITE);
            }
        }
    }

    cs_free(insn, n);
    cs_close(&h);
    return 1;
}

const char* semsig_backend_name(void) {
    return "capstone";
}