vatrog-vm-introspection-engine/src/handlers/semsig_backend_capstone.c

/* semsig_backend_capstone.c - Capstone implementation of the operand-decode seam.
 *
 * The ONE TU that includes Capstone headers and links its library. It implements
 * the semsig_backend.h contract: decode one x86-64 instruction into a sem_insn
 * (length, backend-neutral mnemonic class, normalized operands), and report the
 * backend name. Built ONLY when VMIE_DISASM=capstone (see CMakeLists.txt).
 *
 * Stateless: a Capstone handle is opened/closed on the call stack per decode, no
 * global mutable state - reentrant. Capstone allocates internally for cs_disasm;
 * the buffer is freed before return so there is no leak across the seam.
 */
#include <stdint.h>
#include <stddef.h>
#include <capstone/capstone.h>
#include "semsig_backend.h"

/* Map a Capstone x86 instruction id to a backend-neutral SEM_MN_* class. STATIC
 * table, not a switch forest; unlisted ids fall through to SEM_MN_OTHER (where
 * the core folds the raw opcode). Conditional branches (Jcc) are a family folded
 * by Capstone group, handled before the table lookup. */
typedef struct { unsigned int id; uint16_t cls; } mn_map;

static const mn_map MN_TABLE[] = {
    /* moves */
    { X86_INS_MOV,    SEM_MN_MOV },
    { X86_INS_MOVZX,  SEM_MN_MOV },
    { X86_INS_MOVSX,  SEM_MN_MOV },
    { X86_INS_MOVSXD, SEM_MN_MOV },
    { X86_INS_MOVAPS, SEM_MN_MOV },
    { X86_INS_MOVAPD, SEM_MN_MOV },
    { X86_INS_MOVDQA, SEM_MN_MOV },
    { X86_INS_MOVDQU, SEM_MN_MOV },
    { X86_INS_MOVD,   SEM_MN_MOV },
    { X86_INS_MOVQ,   SEM_MN_MOV },
    { X86_INS_XCHG,   SEM_MN_MOV },
    { X86_INS_LEA,    SEM_MN_LEA },
    /* arithmetic */
    { X86_INS_ADD,    SEM_MN_ARITH },
    { X86_INS_ADC,    SEM_MN_ARITH },
    { X86_INS_SUB,    SEM_MN_ARITH },
    { X86_INS_SBB,    SEM_MN_ARITH },
    { X86_INS_INC,    SEM_MN_ARITH },
    { X86_INS_DEC,    SEM_MN_ARITH },
    { X86_INS_NEG,    SEM_MN_ARITH },
    { X86_INS_MUL,    SEM_MN_ARITH },
    { X86_INS_IMUL,   SEM_MN_ARITH },
    { X86_INS_DIV,    SEM_MN_ARITH },
    { X86_INS_IDIV,   SEM_MN_ARITH },
    /* logic / shifts */
    { X86_INS_AND,    SEM_MN_LOGIC },
    { X86_INS_OR,     SEM_MN_LOGIC },
    { X86_INS_XOR,    SEM_MN_LOGIC },
    { X86_INS_NOT,    SEM_MN_LOGIC },
    { X86_INS_SHL,    SEM_MN_LOGIC },
    { X86_INS_SHR,    SEM_MN_LOGIC },
    { X86_INS_SAR,    SEM_MN_LOGIC },
    { X86_INS_ROL,    SEM_MN_LOGIC },
    { X86_INS_ROR,    SEM_MN_LOGIC },
    /* compare / test */
    { X86_INS_CMP,    SEM_MN_CMP },
    { X86_INS_TEST,   SEM_MN_TEST },
    /* control flow */
    { X86_INS_JMP,    SEM_MN_JMP },
    { X86_INS_CALL,   SEM_MN_CALL },
    { X86_INS_RET,    SEM_MN_RET },
    { X86_INS_RETF,   SEM_MN_RET },
    { X86_INS_RETFQ,  SEM_MN_RET },
    { X86_INS_PUSH,   SEM_MN_PUSH },
    { X86_INS_POP,    SEM_MN_POP },
    { X86_INS_NOP,    SEM_MN_NOP },
};

/* True if `g` is among the instruction's Capstone groups. */
static int has_group(const cs_insn* in, uint8_t g) {
    const cs_detail* d = in->detail;
    if (!d) { return 0; }
    for (uint8_t i = 0; i < d->groups_count; i++) {
        if (d->groups[i] == g) { return 1; }
    }
    return 0;
}

static uint16_t classify_mnemonic(const cs_insn* in) {
    if (has_group(in, X86_GRP_CALL)) { return SEM_MN_CALL; }
    if (has_group(in, X86_GRP_RET))  { return SEM_MN_RET; }
    if (has_group(in, X86_GRP_JUMP)) {
        /* unconditional jmp vs conditional jcc: JMP id is unconditional. */
        return (in->id == X86_INS_JMP) ? SEM_MN_JMP : SEM_MN_BRANCH;
    }
    for (size_t i = 0; i < sizeof MN_TABLE / sizeof MN_TABLE[0]; i++) {
        if (MN_TABLE[i].id == in->id) { return MN_TABLE[i].cls; }
    }
    if (has_group(in, X86_GRP_FPU)) { return SEM_MN_FLOAT; }
    if (has_group(in, X86_GRP_SSE) || has_group(in, X86_GRP_SSE1) ||
        has_group(in, X86_GRP_SSE2)) { return SEM_MN_VEC; }
    return SEM_MN_OTHER;
}

/* Map a Capstone x86 register id to a canonical SEM_RC_* class. Capstone has no
 * single "register class" accessor; classify by the documented id ranges. */
static uint8_t reg_class_of(unsigned int reg) {
    if (reg == X86_REG_INVALID) { return SEM_RC_NONE; }
    if ((reg >= X86_REG_AH && reg <= X86_REG_BH) ||
        (reg >= X86_REG_AL && reg <= X86_REG_R15D) ||
        (reg >= X86_REG_RAX && reg <= X86_REG_RBP) ||
        (reg >= X86_REG_RSP && reg <= X86_REG_RDI) ||
        (reg >= X86_REG_R8 && reg <= X86_REG_R15)) { return SEM_RC_GPR; }
    if (reg >= X86_REG_XMM0 && reg <= X86_REG_ZMM31) { return SEM_RC_XMM; }
    if (reg >= X86_REG_K0 && reg <= X86_REG_K7) { return SEM_RC_MASK; }
    if (reg >= X86_REG_CS && reg <= X86_REG_SS) { return SEM_RC_SEG; }
    if (reg >= X86_REG_EFLAGS && reg <= X86_REG_RIP) { return SEM_RC_FLAGS; }
    return SEM_RC_OTHER;
}

static uint8_t width_log2_of(uint8_t size_bytes) {
    uint8_t l = 0;
    uint8_t v = size_bytes;
    while (v > 1u && l < 4u) { v >>= 1; l++; }
    return l;
}

int semsig_backend_decode(const uint8_t* code, size_t avail, sem_insn* out) {
    if (!code || !out || avail == 0) { return 0; }

    csh h;
    if (cs_open(CS_ARCH_X86, CS_MODE_64, &h) != CS_ERR_OK) { return 0; }
    cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);

    cs_insn* insn = NULL;
    const size_t n = cs_disasm(h, code, avail, 0, 1, &insn);
    if (n < 1 || !insn) {
        if (insn) { cs_free(insn, n); }
        cs_close(&h);
        return 0;
    }

    for (size_t i = 0; i < sizeof *out; i++) { ((uint8_t*)out)[i] = 0; }
    out->length         = (uint8_t)insn->size;
    out->mnemonic_class = classify_mnemonic(insn);
    out->is_control_flow =
        (has_group(insn, X86_GRP_JUMP) || has_group(insn, X86_GRP_CALL) ||
         has_group(insn, X86_GRP_RET)) ? 1u : 0u;
    out->raw_opcode = (out->mnemonic_class == SEM_MN_OTHER && insn->detail)
                      ? insn->detail->x86.opcode[0] : 0u;

    uint8_t no = 0;
    if (insn->detail) {
        const cs_x86* x = &insn->detail->x86;
        for (uint8_t i = 0; i < x->op_count && no < 4; i++) {
            const cs_x86_op* o = &x->operands[i];
            switch (o->type) {
                case X86_OP_REG:
                    out->op[no].kind       = SEMOP_REG;
                    out->op[no].reg_class  = reg_class_of(o->reg);
                    out->op[no].width_log2 = width_log2_of(o->size);
                    no++;
                    break;
                case X86_OP_MEM:
                    out->op[no].kind          = SEMOP_MEM;
                    out->op[no].width_log2    = width_log2_of(o->size);
                    out->op[no].mem_has_base  =
                        (o->mem.base != X86_REG_INVALID &&
                         o->mem.base != X86_REG_RIP) ? 1u : 0u;
                    out->op[no].mem_has_index =
                        (o->mem.index != X86_REG_INVALID) ? 1u : 0u;
                    out->op[no].mem_is_riprel =
                        (o->mem.base == X86_REG_RIP) ? 1u : 0u;
                    no++;
                    break;
                case X86_OP_IMM:
                    out->op[no].kind       = SEMOP_IMM;
                    out->op[no].width_log2 = width_log2_of(o->size);
                    no++;
                    break;
                default:
                    break;
            }
        }
    }
    out->noperands = no;

    cs_free(insn, n);
    cs_close(&h);
    return 1;
}

const char* semsig_backend_name(void) {
    return "capstone";
}