lirent/vatrog-vm-introspection-engine
1
0
Fork 0
mirror of https://dev.lirent.ru/Vatrog/vm-introspection-engine.git synced 2026-07-08 23:46:36 +03:00
Code Issues Packages Projects Releases Wiki Activity
Files
7cff67afce485bba8097bf60ed412e95f550e3ce
vatrog-vm-introspection-engine/src/handlers/semsig_backend_capstone.c
T
lirent 244511b66e Add readable operand decode and register def-use analysis 
insndec.h: insn_decode() decodes one instruction into a readable form -
concrete registers as canonical roots (al/ah/ax/eax/rax -> one root), memory
base/index/scale/disp, immediate values, operand sizes, per-operand read/write,
and implicit operands/effects (rsp on push/pop, rdx:rax on mul/div, FLAGS). It
is ABI-agnostic: a call reports only its architectural effects.

codetrace.h: func_usedef(fn, abi, ...) computes per-instruction register
use/def over a function view (reusing cfg_blocks), splitting writes into
full-kill / partial / conditional so a reaching-definitions pass can be built on
top. With TRACE_ABI_WIN64 it marks the Win64 caller-saved set
(rax,rcx,rdx,r8-r11,xmm0-5,flags) clobbered at each call site; the clobber table
is confined to codetrace.c and the generic layer stays OS-agnostic
(TRACE_ABI_NONE keeps the conservative behaviour).

Both ride the existing optional disassembler seam (a second insn_decode_full
is added to semsig_backend.h; sem_insn and semsig_hash are untouched), gated
behind VMIE_HAVE_DISASM; OFF builds a stub. Adds the vmie_win32_func_usedef
wrapper. No new build option - the feature shares VMIE_DISASM with semsig.
2026-06-28 23:02:40 +03:00

441 lines
18 KiB
C
Raw Blame History

/* semsig_backend_capstone.c - Capstone implementation of the operand-decode seam.
*
* The ONE TU that includes Capstone headers and links its library. It implements
* the semsig_backend.h contract: decode one x86-64 instruction into a sem_insn
* (length, backend-neutral mnemonic class, normalized operands), and report the
* backend name. Built ONLY when VMIE_DISASM=capstone (see CMakeLists.txt).
*
* Stateless: a Capstone handle is opened/closed on the call stack per decode, no
* global mutable state - reentrant. Capstone allocates internally for cs_disasm;
* the buffer is freed before return so there is no leak across the seam.
*/
#include <stdint.h>
#include <stddef.h>
#include <capstone/capstone.h>
#include "semsig_backend.h"
/* Map a Capstone x86 instruction id to a backend-neutral SEM_MN_* class. STATIC
* table, not a switch forest; unlisted ids fall through to SEM_MN_OTHER (where
* the core folds the raw opcode). Conditional branches (Jcc) are a family folded
* by Capstone group, handled before the table lookup. */
typedef struct { unsigned int id; uint16_t cls; } mn_map;
static const mn_map MN_TABLE[] = {
/* moves */
{ X86_INS_MOV, SEM_MN_MOV },
{ X86_INS_MOVZX, SEM_MN_MOV },
{ X86_INS_MOVSX, SEM_MN_MOV },
{ X86_INS_MOVSXD, SEM_MN_MOV },
{ X86_INS_MOVAPS, SEM_MN_MOV },
{ X86_INS_MOVAPD, SEM_MN_MOV },
{ X86_INS_MOVDQA, SEM_MN_MOV },
{ X86_INS_MOVDQU, SEM_MN_MOV },
{ X86_INS_MOVD, SEM_MN_MOV },
{ X86_INS_MOVQ, SEM_MN_MOV },
{ X86_INS_XCHG, SEM_MN_MOV },
{ X86_INS_LEA, SEM_MN_LEA },
/* arithmetic */
{ X86_INS_ADD, SEM_MN_ARITH },
{ X86_INS_ADC, SEM_MN_ARITH },
{ X86_INS_SUB, SEM_MN_ARITH },
{ X86_INS_SBB, SEM_MN_ARITH },
{ X86_INS_INC, SEM_MN_ARITH },
{ X86_INS_DEC, SEM_MN_ARITH },
{ X86_INS_NEG, SEM_MN_ARITH },
{ X86_INS_MUL, SEM_MN_ARITH },
{ X86_INS_IMUL, SEM_MN_ARITH },
{ X86_INS_DIV, SEM_MN_ARITH },
{ X86_INS_IDIV, SEM_MN_ARITH },
/* logic / shifts */
{ X86_INS_AND, SEM_MN_LOGIC },
{ X86_INS_OR, SEM_MN_LOGIC },
{ X86_INS_XOR, SEM_MN_LOGIC },
{ X86_INS_NOT, SEM_MN_LOGIC },
{ X86_INS_SHL, SEM_MN_LOGIC },
{ X86_INS_SHR, SEM_MN_LOGIC },
{ X86_INS_SAR, SEM_MN_LOGIC },
{ X86_INS_ROL, SEM_MN_LOGIC },
{ X86_INS_ROR, SEM_MN_LOGIC },
/* compare / test */
{ X86_INS_CMP, SEM_MN_CMP },
{ X86_INS_TEST, SEM_MN_TEST },
/* control flow */
{ X86_INS_JMP, SEM_MN_JMP },
{ X86_INS_CALL, SEM_MN_CALL },
{ X86_INS_RET, SEM_MN_RET },
{ X86_INS_RETF, SEM_MN_RET },
{ X86_INS_RETFQ, SEM_MN_RET },
{ X86_INS_PUSH, SEM_MN_PUSH },
{ X86_INS_POP, SEM_MN_POP },
{ X86_INS_NOP, SEM_MN_NOP },
};
/* True if `g` is among the instruction's Capstone groups. */
static int has_group(const cs_insn* in, uint8_t g) {
const cs_detail* d = in->detail;
if (!d) { return 0; }
for (uint8_t i = 0; i < d->groups_count; i++) {
if (d->groups[i] == g) { return 1; }
}
return 0;
}
static uint16_t classify_mnemonic(const cs_insn* in) {
if (has_group(in, X86_GRP_CALL)) { return SEM_MN_CALL; }
if (has_group(in, X86_GRP_RET)) { return SEM_MN_RET; }
if (has_group(in, X86_GRP_JUMP)) {
/* unconditional jmp vs conditional jcc: JMP id is unconditional. */
return (in->id == X86_INS_JMP) ? SEM_MN_JMP : SEM_MN_BRANCH;
}
for (size_t i = 0; i < sizeof MN_TABLE / sizeof MN_TABLE[0]; i++) {
if (MN_TABLE[i].id == in->id) { return MN_TABLE[i].cls; }
}
if (has_group(in, X86_GRP_FPU)) { return SEM_MN_FLOAT; }
if (has_group(in, X86_GRP_SSE) || has_group(in, X86_GRP_SSE1) ||
has_group(in, X86_GRP_SSE2)) { return SEM_MN_VEC; }
return SEM_MN_OTHER;
}
/* Map a Capstone x86 register id to a canonical SEM_RC_* class. Capstone has no
* single "register class" accessor; classify by the documented id ranges. */
static uint8_t reg_class_of(unsigned int reg) {
if (reg == X86_REG_INVALID) { return SEM_RC_NONE; }
if ((reg >= X86_REG_AH && reg <= X86_REG_BH) ||
(reg >= X86_REG_AL && reg <= X86_REG_R15D) ||
(reg >= X86_REG_RAX && reg <= X86_REG_RBP) ||
(reg >= X86_REG_RSP && reg <= X86_REG_RDI) ||
(reg >= X86_REG_R8 && reg <= X86_REG_R15)) { return SEM_RC_GPR; }
if (reg >= X86_REG_XMM0 && reg <= X86_REG_ZMM31) { return SEM_RC_XMM; }
if (reg >= X86_REG_K0 && reg <= X86_REG_K7) { return SEM_RC_MASK; }
if (reg >= X86_REG_CS && reg <= X86_REG_SS) { return SEM_RC_SEG; }
if (reg >= X86_REG_EFLAGS && reg <= X86_REG_RIP) { return SEM_RC_FLAGS; }
return SEM_RC_OTHER;
}
static uint8_t width_log2_of(uint8_t size_bytes) {
uint8_t l = 0;
uint8_t v = size_bytes;
while (v > 1u && l < 4u) { v >>= 1; l++; }
return l;
}
int semsig_backend_decode(const uint8_t* code, size_t avail, sem_insn* out) {
if (!code || !out || avail == 0) { return 0; }
csh h;
if (cs_open(CS_ARCH_X86, CS_MODE_64, &h) != CS_ERR_OK) { return 0; }
cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);
cs_insn* insn = NULL;
const size_t n = cs_disasm(h, code, avail, 0, 1, &insn);
if (n < 1 || !insn) {
if (insn) { cs_free(insn, n); }
cs_close(&h);
return 0;
}
for (size_t i = 0; i < sizeof *out; i++) { ((uint8_t*)out)[i] = 0; }
out->length = (uint8_t)insn->size;
out->mnemonic_class = classify_mnemonic(insn);
out->is_control_flow =
(has_group(insn, X86_GRP_JUMP) || has_group(insn, X86_GRP_CALL) ||
has_group(insn, X86_GRP_RET)) ? 1u : 0u;
out->raw_opcode = (out->mnemonic_class == SEM_MN_OTHER && insn->detail)
? insn->detail->x86.opcode[0] : 0u;
uint8_t no = 0;
if (insn->detail) {
const cs_x86* x = &insn->detail->x86;
for (uint8_t i = 0; i < x->op_count && no < 4; i++) {
const cs_x86_op* o = &x->operands[i];
switch (o->type) {
case X86_OP_REG:
out->op[no].kind = SEMOP_REG;
out->op[no].reg_class = reg_class_of(o->reg);
out->op[no].width_log2 = width_log2_of(o->size);
no++;
break;
case X86_OP_MEM:
out->op[no].kind = SEMOP_MEM;
out->op[no].width_log2 = width_log2_of(o->size);
out->op[no].mem_has_base =
(o->mem.base != X86_REG_INVALID &&
o->mem.base != X86_REG_RIP) ? 1u : 0u;
out->op[no].mem_has_index =
(o->mem.index != X86_REG_INVALID) ? 1u : 0u;
out->op[no].mem_is_riprel =
(o->mem.base == X86_REG_RIP) ? 1u : 0u;
no++;
break;
case X86_OP_IMM:
out->op[no].kind = SEMOP_IMM;
out->op[no].width_log2 = width_log2_of(o->size);
no++;
break;
default:
break;
}
}
}
out->noperands = no;
cs_free(insn, n);
cs_close(&h);
return 1;
}
/* ---- rich decode (insn_decode_full): the lossless projection ------------- */
/* Map a Capstone x86 register id to a canonical reg_id ROOT (al/ax/eax/rax ->
* REGID_RAX, xmm/ymm/zmm -> a REGID_VEC* root, k0..k7 -> REGID_K*). Capstone has
* no "largest enclosing" helper, so fold by the documented id ranges (the same
* approach reg_class_of already uses). The raw x86_reg never leaves this TU. */
static uint8_t reg_root_of(unsigned int reg) {
switch (reg) {
case X86_REG_AL: case X86_REG_AH: case X86_REG_AX:
case X86_REG_EAX: case X86_REG_RAX: return REGID_RAX;
case X86_REG_CL: case X86_REG_CH: case X86_REG_CX:
case X86_REG_ECX: case X86_REG_RCX: return REGID_RCX;
case X86_REG_DL: case X86_REG_DH: case X86_REG_DX:
case X86_REG_EDX: case X86_REG_RDX: return REGID_RDX;
case X86_REG_BL: case X86_REG_BH: case X86_REG_BX:
case X86_REG_EBX: case X86_REG_RBX: return REGID_RBX;
case X86_REG_SPL: case X86_REG_SP: case X86_REG_ESP:
case X86_REG_RSP: return REGID_RSP;
case X86_REG_BPL: case X86_REG_BP: case X86_REG_EBP:
case X86_REG_RBP: return REGID_RBP;
case X86_REG_SIL: case X86_REG_SI: case X86_REG_ESI:
case X86_REG_RSI: return REGID_RSI;
case X86_REG_DIL: case X86_REG_DI: case X86_REG_EDI:
case X86_REG_RDI: return REGID_RDI;
case X86_REG_R8B: case X86_REG_R8W: case X86_REG_R8D:
case X86_REG_R8: return REGID_R8;
case X86_REG_R9B: case X86_REG_R9W: case X86_REG_R9D:
case X86_REG_R9: return REGID_R9;
case X86_REG_R10B: case X86_REG_R10W: case X86_REG_R10D:
case X86_REG_R10: return REGID_R10;
case X86_REG_R11B: case X86_REG_R11W: case X86_REG_R11D:
case X86_REG_R11: return REGID_R11;
case X86_REG_R12B: case X86_REG_R12W: case X86_REG_R12D:
case X86_REG_R12: return REGID_R12;
case X86_REG_R13B: case X86_REG_R13W: case X86_REG_R13D:
case X86_REG_R13: return REGID_R13;
case X86_REG_R14B: case X86_REG_R14W: case X86_REG_R14D:
case X86_REG_R14: return REGID_R14;
case X86_REG_R15B: case X86_REG_R15W: case X86_REG_R15D:
case X86_REG_R15: return REGID_R15;
default: break;
}
if (reg >= X86_REG_XMM0 && reg <= X86_REG_XMM31) {
return (uint8_t)(REGID_VEC0 + (reg - X86_REG_XMM0));
}
if (reg >= X86_REG_YMM0 && reg <= X86_REG_YMM31) {
return (uint8_t)(REGID_VEC0 + (reg - X86_REG_YMM0));
}
if (reg >= X86_REG_ZMM0 && reg <= X86_REG_ZMM31) {
return (uint8_t)(REGID_VEC0 + (reg - X86_REG_ZMM0));
}
if (reg >= X86_REG_K0 && reg <= X86_REG_K7) {
return (uint8_t)(REGID_K0 + (reg - X86_REG_K0));
}
if (reg == X86_REG_EFLAGS) { return REGID_FLAGS; }
if (reg == X86_REG_INVALID) { return REGID_NONE; }
return REGID_OTHER;
}
/* Approximate the ACCESS width of a register in log2 bytes from its sub-register
* id (Capstone register ids encode the width); used when only an id is known
* (cs_regs_access returns ids without sizes). Coarse but enough for overlap. */
static uint8_t reg_width_log2(unsigned int reg) {
switch (reg) {
case X86_REG_AL: case X86_REG_AH: case X86_REG_CL: case X86_REG_CH:
case X86_REG_DL: case X86_REG_DH: case X86_REG_BL: case X86_REG_BH:
case X86_REG_SPL: case X86_REG_BPL: case X86_REG_SIL: case X86_REG_DIL:
case X86_REG_R8B: case X86_REG_R9B: case X86_REG_R10B:
case X86_REG_R11B: case X86_REG_R12B: case X86_REG_R13B:
case X86_REG_R14B: case X86_REG_R15B: return 0;
case X86_REG_AX: case X86_REG_CX: case X86_REG_DX: case X86_REG_BX:
case X86_REG_SP: case X86_REG_BP: case X86_REG_SI: case X86_REG_DI:
case X86_REG_R8W: case X86_REG_R9W: case X86_REG_R10W:
case X86_REG_R11W: case X86_REG_R12W: case X86_REG_R13W:
case X86_REG_R14W: case X86_REG_R15W: return 1;
case X86_REG_EAX: case X86_REG_ECX: case X86_REG_EDX: case X86_REG_EBX:
case X86_REG_ESP: case X86_REG_EBP: case X86_REG_ESI: case X86_REG_EDI:
case X86_REG_R8D: case X86_REG_R9D: case X86_REG_R10D:
case X86_REG_R11D: case X86_REG_R12D: case X86_REG_R13D:
case X86_REG_R14D: case X86_REG_R15D: return 2;
default: break;
}
if (reg >= X86_REG_XMM0 && reg <= X86_REG_XMM31) { return 4; }
return 3; /* 64-bit GPR / vector-as-root / other: default qword */
}
/* Backend-neutral mnemonic CLASS for the rich decode (insn_class enum), by
* translating the existing classify_mnemonic SEM_MN_* result. */
static uint8_t insn_class_of(const cs_insn* in) {
switch (classify_mnemonic(in)) {
case SEM_MN_MOV: return INSN_MOV;
case SEM_MN_ARITH: return INSN_ARITH;
case SEM_MN_LOGIC: return INSN_LOGIC;
case SEM_MN_CMP: return INSN_CMP;
case SEM_MN_TEST: return INSN_TEST;
case SEM_MN_BRANCH: return INSN_BRANCH;
case SEM_MN_JMP: return INSN_JMP;
case SEM_MN_CALL: return INSN_CALL;
case SEM_MN_RET: return INSN_RET;
case SEM_MN_PUSH: return INSN_PUSH;
case SEM_MN_POP: return INSN_POP;
case SEM_MN_LEA: return INSN_LEA;
case SEM_MN_NOP: return INSN_NOP;
case SEM_MN_FLOAT: return INSN_FLOAT;
case SEM_MN_VEC: return INSN_VEC;
default: return INSN_OTHER;
}
}
/* Normalize a Capstone cs_ac_type access mask to a reg_access. */
static uint8_t access_of(uint8_t a) {
const int r = (a & CS_AC_READ) != 0;
const int w = (a & CS_AC_WRITE) != 0;
if (r && w) { return ACC_READWRITE; }
if (r) { return ACC_READ; }
if (w) { return ACC_WRITE; }
return ACC_NONE;
}
/* writes_zero_extend rule (see insndec.h): 32-bit GPR write zero-extends; EVEX/
* VEX vector write zero-extends the upper part; 8/16-bit and legacy-SSE are
* partial. */
static uint8_t zx_of(uint8_t root, uint8_t width_log2, int vector_evex_vex) {
if (root >= REGID_RAX && root <= REGID_R15) {
return (width_log2 == 2) ? 1u : 0u;
}
if (root >= REGID_VEC0 && root <= REGID_VEC31) {
return vector_evex_vex ? 1u : 0u;
}
return 0u;
}
static void regset_add(reg_ref* set, uint8_t* n, uint8_t root,
uint8_t width_log2, uint8_t zx, uint8_t access) {
if (root == REGID_NONE) { return; }
for (uint8_t i = 0; i < *n && i < INSN_MAX_REGREFS; i++) {
if (set[i].root == root) {
if (width_log2 > set[i].width_log2) {
set[i].width_log2 = width_log2;
set[i].writes_zero_extend = zx;
}
return;
}
}
if (*n < INSN_MAX_REGREFS) {
set[*n].root = root;
set[*n].width_log2 = width_log2;
set[*n].writes_zero_extend = zx;
set[*n].access = access;
}
(*n)++;
}
int insn_decode_full(const uint8_t* code, size_t avail, insn_decoded* out) {
if (!code || !out || avail == 0) { return 0; }
csh h;
if (cs_open(CS_ARCH_X86, CS_MODE_64, &h) != CS_ERR_OK) { return 0; }
cs_option(h, CS_OPT_DETAIL, CS_OPT_ON);
cs_insn* insn = NULL;
const size_t n = cs_disasm(h, code, avail, 0, 1, &insn);
if (n < 1 || !insn) {
if (insn) { cs_free(insn, n); }
cs_close(&h);
return 0;
}
for (size_t i = 0; i < sizeof *out; i++) { ((uint8_t*)out)[i] = 0; }
out->length = (uint8_t)insn->size;
out->mnemonic_class = insn_class_of(insn);
out->is_control_flow =
(has_group(insn, X86_GRP_JUMP) || has_group(insn, X86_GRP_CALL) ||
has_group(insn, X86_GRP_RET)) ? 1u : 0u;
if (insn->detail) {
const cs_x86* x = &insn->detail->x86;
/* A VEX/EVEX form zero-extends the upper part of a written vector root;
* legacy-SSE preserves it. Detect by the VEX/EVEX leading opcode byte. */
const int is_vex_evex =
(x->opcode[0] == 0xC4 || x->opcode[0] == 0xC5 || x->opcode[0] == 0x62);
/* Explicit operands (readable view): op[]. */
uint8_t no = 0;
for (uint8_t i = 0; i < x->op_count && no < INSN_MAX_OPERANDS; i++) {
const cs_x86_op* o = &x->operands[i];
const uint8_t acc = access_of(o->access);
const uint8_t wl = width_log2_of(o->size);
switch (o->type) {
case X86_OP_REG: {
const uint8_t root = reg_root_of(o->reg);
out->op[no].kind = OPND_REG;
out->op[no].access = acc;
out->op[no].width_log2 = wl;
out->op[no].reg_root = root;
out->op[no].reg_zero_extend = zx_of(root, wl, is_vex_evex);
no++;
break;
}
case X86_OP_MEM:
out->op[no].kind = OPND_MEM;
out->op[no].access = acc;
out->op[no].width_log2 = wl;
out->op[no].mem_base =
(o->mem.base == X86_REG_RIP) ? REGID_NONE
: reg_root_of(o->mem.base);
out->op[no].mem_index = reg_root_of(o->mem.index);
out->op[no].mem_scale = (uint8_t)o->mem.scale;
out->op[no].mem_is_riprel =
(o->mem.base == X86_REG_RIP) ? 1u : 0u;
out->op[no].mem_disp = o->mem.disp;
no++;
break;
case X86_OP_IMM:
out->op[no].kind = OPND_IMM;
out->op[no].access = ACC_READ;
out->op[no].imm_width_log2 = wl;
out->op[no].imm_value = o->imm;
no++;
break;
default:
break;
}
}
out->noperands = no;
/* Aggregate implicit-or-explicit register sets via cs_regs_access: the
* FULL read/write sets including rsp/rdx:rax/EFLAGS. EFLAGS -> REGID_FLAGS;
* vector writes carry zero-extend by form. */
cs_regs rd, wr;
uint8_t nrd = 0, nwr = 0;
if (cs_regs_access(h, insn, rd, &nrd, wr, &nwr) == CS_ERR_OK) {
for (uint8_t i = 0; i < nrd; i++) {
const uint8_t root = reg_root_of(rd[i]);
regset_add(out->reads, &out->nreads, root,
reg_width_log2(rd[i]), 0, ACC_READ);
}
for (uint8_t i = 0; i < nwr; i++) {
const uint8_t root = reg_root_of(wr[i]);
const uint8_t wl = reg_width_log2(wr[i]);
regset_add(out->writes, &out->nwrites, root, wl,
zx_of(root, wl, is_vex_evex), ACC_WRITE);
}
}
}
cs_free(insn, n);
cs_close(&h);
return 1;
}
const char* semsig_backend_name(void) {
return "capstone";
}
Reference in New Issue View Git Blame Copy Permalink