From 0e4b3c7fe6f01533001814be37676e3b810978c0 Mon Sep 17 00:00:00 2001 From: Will Hawkins Date: Fri, 17 Jan 2025 17:59:27 -0500 Subject: [PATCH] Cleanup Handling of Targets During JITing. (#626) Until now, special targets during JITing were handled using constant valudes that _could_ be valid addresses on certain architectures. To disambiguate, this patch adds more precise metadata for each target. Signed-off-by: Will Hawkins --- vm/ubpf_jit_arm64.c | 107 +++++++++++------- vm/ubpf_jit_support.c | 41 +++---- vm/ubpf_jit_support.h | 124 ++++++++++++-------- vm/ubpf_jit_x86_64.c | 255 +++++++++++++++++++++++------------------- 4 files changed, 301 insertions(+), 226 deletions(-) diff --git a/vm/ubpf_jit_arm64.c b/vm/ubpf_jit_arm64.c index 5353c3b06..4682c8e4d 100644 --- a/vm/ubpf_jit_arm64.c +++ b/vm/ubpf_jit_arm64.c @@ -206,7 +206,8 @@ emit_addsub_immediate( sh = imm_shift_on; } assert(imm12 < imm_unshifted_max); - emit_instruction(state, sz(sixty_four) | sh | (op << 29) | imm_op_base | (0 << 22) | (imm12 << 10) | (rn << 5) | rd); + emit_instruction( + state, sz(sixty_four) | sh | (op << 29) | imm_op_base | (0 << 22) | (imm12 << 10) | (rn << 5) | rd); } /* [ArmARM-A H.a]: C4.1.67: Add/subtract (shifted register). */ @@ -263,7 +264,8 @@ emit_loadstore_register( } static void -emit_loadstore_literal(struct jit_state* state, enum LoadStoreOpcode op, enum Registers rt, uint32_t target) +emit_loadstore_literal( + struct jit_state* state, enum LoadStoreOpcode op, enum Registers rt, struct PatchableTarget target) { note_load(state, target); const uint32_t reg_op_base = 0x08000000U; @@ -271,9 +273,9 @@ emit_loadstore_literal(struct jit_state* state, enum LoadStoreOpcode op, enum Re } static void -emit_adr(struct jit_state* state, uint32_t offset, enum Registers rd) +emit_adr(struct jit_state* state, struct PatchableTarget target, enum Registers rd) { - note_lea(state, offset); + note_lea(state, target); uint32_t instr = 0x10000000 | rd; emit_instruction(state, instr); } @@ -355,17 +357,17 @@ enum UnconditionalBranchImmediateOpcode /* [ArmARM-A H.a]: C4.1.65: Unconditional branch (immediate). */ static uint32_t emit_unconditionalbranch_immediate( - struct jit_state* state, enum UnconditionalBranchImmediateOpcode op, int32_t target_pc) + struct jit_state* state, enum UnconditionalBranchImmediateOpcode op, struct PatchableTarget target) { uint32_t source_offset = state->offset; struct patchable_relative* table = state->jumps; int* num_jumps = &state->num_jumps; - if (op == UBR_BL && target_pc != TARGET_PC_ENTER) { + if (op == UBR_BL && !target.is_special) { table = state->local_calls; num_jumps = &state->num_local_calls; } - emit_patchable_relative(state->offset, target_pc, 0, table, (*num_jumps)++); + emit_patchable_relative(table, state->offset, target, (*num_jumps)++); emit_instruction(state, op); return source_offset; @@ -400,10 +402,10 @@ enum ConditionalBranchImmediateOpcode /* [ArmARM-A H.a]: C4.1.65: Conditional branch (immediate). */ static uint32_t -emit_conditionalbranch_immediate(struct jit_state* state, enum Condition cond, uint32_t target_pc) +emit_conditionalbranch_immediate(struct jit_state* state, enum Condition cond, struct PatchableTarget target) { uint32_t source_offset = state->offset; - emit_patchable_relative(state->offset, target_pc, 0, state->jumps, state->num_jumps++); + emit_patchable_relative(state->jumps, state->offset, target, state->num_jumps++); emit_instruction(state, BR_Bcond | (0 << 5) | cond); return source_offset; } @@ -565,8 +567,10 @@ emit_jit_prologue(struct jit_state* state, size_t ubpf_stack_size) /* Copy R0 to the volatile context for safe keeping. */ emit_logical_register(state, true, LOG_ORR, VOLATILE_CTXT, RZ, R0); - emit_unconditionalbranch_immediate(state, UBR_BL, TARGET_PC_ENTER); - emit_unconditionalbranch_immediate(state, UBR_B, TARGET_PC_EXIT); + DECLARE_PATCHABLE_SPECIAL_TARGET(exit_tgt, Exit); + DECLARE_PATCHABLE_SPECIAL_TARGET(enter_tgt, Enter); + emit_unconditionalbranch_immediate(state, UBR_BL, enter_tgt); + emit_unconditionalbranch_immediate(state, UBR_B, exit_tgt); state->entry_loc = state->offset; } @@ -616,13 +620,15 @@ emit_dispatched_external_helper_call(struct jit_state* state, struct ubpf_vm* vm // Determine whether to call it through a dispatcher or by index and then load up the address // of that function. - emit_loadstore_literal(state, LS_LDRL, temp_register, TARGET_PC_EXTERNAL_DISPATCHER); + DECLARE_PATCHABLE_SPECIAL_TARGET(external_dispatcher_pt, ExternalDispatcher); + emit_loadstore_literal(state, LS_LDRL, temp_register, external_dispatcher_pt); // Check whether temp_register is empty. emit_addsub_immediate(state, true, AS_SUBS, temp_register, temp_register, 0); // Jump to the call if we are ready to roll (because we are using an external dispatcher). - uint32_t external_dispatcher_jump_source = emit_conditionalbranch_immediate(state, COND_NE, 0); + DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(default_tgt, 0); + uint32_t external_dispatcher_jump_source = emit_conditionalbranch_immediate(state, COND_NE, default_tgt); // We are not ready to roll. In other words, we are going to load the helper function address by index. emit_movewide_immediate(state, true, R5, idx); @@ -630,7 +636,8 @@ emit_dispatched_external_helper_call(struct jit_state* state, struct ubpf_vm* vm emit_dataprocessing_twosource(state, true, DP2_LSLV, R5, R5, R6); emit_movewide_immediate(state, true, temp_register, 0); - emit_adr(state, TARGET_LOAD_HELPER_TABLE, temp_register); + DECLARE_PATCHABLE_SPECIAL_TARGET(load_helper_tgt, LoadHelperTable); + emit_adr(state, load_helper_tgt, temp_register); emit_addsub_register(state, true, AS_ADD, temp_register, temp_register, R5); emit_loadstore_immediate(state, LS_LDRX, temp_register, temp_register, 0); @@ -640,7 +647,7 @@ emit_dispatched_external_helper_call(struct jit_state* state, struct ubpf_vm* vm // And now we, too, are ready to roll. So, let's jump around the code that sets up the additional // parameters for the external dispatcher. We will end up at the call site where both paths // will rendezvous. - uint32_t no_dispatcher_jump_source = emit_unconditionalbranch_immediate(state, UBR_B, 0); + uint32_t no_dispatcher_jump_source = emit_unconditionalbranch_immediate(state, UBR_B, default_tgt); // Mark the landing spot for the jump around the code that sets up a call to a helper function // when no external dispatcher is present. @@ -685,7 +692,8 @@ emit_local_call(struct jit_state* state, uint32_t target_pc) emit_loadstorepair_immediate(state, LSP_STPX, map_register(6), map_register(7), SP, 16); emit_loadstorepair_immediate(state, LSP_STPX, map_register(8), map_register(9), SP, 32); - emit_unconditionalbranch_immediate(state, UBR_BL, target_pc); + DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(tgt, target_pc); + emit_unconditionalbranch_immediate(state, UBR_BL, tgt); emit_loadstore_immediate(state, LS_LDRX, R30, SP, 0); emit_loadstore_immediate(state, LS_LDRX, temp_register, SP, 8); @@ -1031,7 +1039,8 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) if (i != 0 && vm->int_funcs[i]) { struct ebpf_inst prev_inst = ubpf_fetch_instruction(vm, i - 1); if (ubpf_instruction_has_fallthrough(prev_inst)) { - fallthrough_jump_source = emit_unconditionalbranch_immediate(state, UBR_B, 0); + DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(default_tgt, 0) + fallthrough_jump_source = emit_unconditionalbranch_immediate(state, UBR_B, default_tgt); fallthrough_jump_present = true; } } @@ -1050,7 +1059,8 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) } if (fallthrough_jump_present) { - fixup_jump_target(state->jumps, state->num_jumps, fallthrough_jump_source, state->offset); + DECLARE_PATCHABLE_REGULAR_JIT_TARGET(fallthrough_tgt, state->offset) + modify_patchable_relatives_target(state->jumps, state->num_jumps, fallthrough_jump_source, fallthrough_tgt); } state->pc_locs[i] = state->offset; @@ -1060,6 +1070,8 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) uint8_t opcode = inst.opcode; uint32_t target_pc = i + inst.offset + 1; + DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(tgt, target_pc); + int sixty_four = is_alu64_op(&inst); // If this is an operation with an immediate operand (and that immediate @@ -1151,7 +1163,7 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) /* TODO use 8 bit immediate when possible */ case EBPF_OP_JA: - emit_unconditionalbranch_immediate(state, UBR_B, target_pc); + emit_unconditionalbranch_immediate(state, UBR_B, tgt); break; case EBPF_OP_JEQ_IMM: case EBPF_OP_JGT_IMM: @@ -1174,7 +1186,7 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) case EBPF_OP_JSLT32_IMM: case EBPF_OP_JSLE32_IMM: emit_addsub_immediate(state, sixty_four, AS_SUBS, RZ, dst, inst.imm); - emit_conditionalbranch_immediate(state, to_condition(opcode), target_pc); + emit_conditionalbranch_immediate(state, to_condition(opcode), tgt); break; case EBPF_OP_JEQ_REG: case EBPF_OP_JGT_REG: @@ -1197,27 +1209,29 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) case EBPF_OP_JSLT32_REG: case EBPF_OP_JSLE32_REG: emit_addsub_register(state, sixty_four, AS_SUBS, RZ, dst, src); - emit_conditionalbranch_immediate(state, to_condition(opcode), target_pc); + emit_conditionalbranch_immediate(state, to_condition(opcode), tgt); break; case EBPF_OP_JSET_REG: case EBPF_OP_JSET32_REG: emit_logical_register(state, sixty_four, LOG_ANDS, RZ, dst, src); - emit_conditionalbranch_immediate(state, to_condition(opcode), target_pc); + emit_conditionalbranch_immediate(state, to_condition(opcode), tgt); break; - case EBPF_OP_CALL: + case EBPF_OP_CALL: { + DECLARE_PATCHABLE_SPECIAL_TARGET(exit_tgt, Exit); if (inst.src == 0) { emit_dispatched_external_helper_call(state, vm, inst.imm); if (inst.imm == vm->unwind_stack_extension_index) { emit_addsub_immediate(state, true, AS_SUBS, RZ, map_register(0), 0); - emit_conditionalbranch_immediate(state, COND_EQ, TARGET_PC_EXIT); + emit_conditionalbranch_immediate(state, COND_EQ, exit_tgt); } } else if (inst.src == 1) { uint32_t call_target = i + inst.imm + 1; emit_local_call(state, call_target); } else { - emit_unconditionalbranch_immediate(state, UBR_B, TARGET_PC_EXIT); + emit_unconditionalbranch_immediate(state, UBR_B, exit_tgt); } break; + } case EBPF_OP_EXIT: emit_addsub_immediate(state, true, AS_ADD, SP, SP, 16); emit_unconditionalbranch_register(state, BR_RET, R30); @@ -1395,14 +1409,26 @@ resolve_jumps(struct jit_state* state) struct patchable_relative jump = state->jumps[i]; int32_t target_loc; - if (jump.target_offset != 0) { - target_loc = jump.target_offset; - } else if (jump.target_pc == TARGET_PC_EXIT) { - target_loc = state->exit_loc; - } else if (jump.target_pc == TARGET_PC_ENTER) { - target_loc = state->entry_loc; + + if (jump.target.is_special) { + // Jumps to special targets Exit and Enter are the only + // valid options. + if (jump.target.target.special == Exit) { + target_loc = state->exit_loc; + } else if (jump.target.target.special == Enter) { + target_loc = state->entry_loc; + } else { + target_loc = -1; + return false; + } + } else { - target_loc = state->pc_locs[jump.target_pc]; + // The jit target, if specified, takes precedence. + if (jump.target.target.regular.jit_target_pc != 0) { + target_loc = jump.target.target.regular.jit_target_pc; + } else { + target_loc = state->pc_locs[jump.target.target.regular.ebpf_target_pc]; + } } int32_t rel = target_loc - jump.offset_loc; @@ -1417,9 +1443,9 @@ resolve_loads(struct jit_state* state) for (unsigned i = 0; i < state->num_loads; ++i) { struct patchable_relative jump = state->loads[i]; - int32_t target_loc; + int32_t target_loc = 0; // Right now it is only possible to load from the external dispatcher. - if (jump.target_pc == TARGET_PC_EXTERNAL_DISPATCHER) { + if (jump.target.is_special && jump.target.target.special == ExternalDispatcher) { target_loc = state->dispatcher_loc; } else { return false; @@ -1439,9 +1465,9 @@ resolve_leas(struct jit_state* state) for (unsigned i = 0; i < state->num_leas; ++i) { struct patchable_relative jump = state->leas[i]; - int32_t target_loc; + int32_t target_loc = 0; // Right now it is only possible to have leas to the helper table. - if (jump.target_pc == TARGET_LOAD_HELPER_TABLE) { + if (jump.target.is_special && jump.target.target.special == LoadHelperTable) { target_loc = state->helper_table_loc; } else { return false; @@ -1461,11 +1487,10 @@ resolve_local_calls(struct jit_state* state) for (unsigned i = 0; i < state->num_local_calls; ++i) { struct patchable_relative local_call = state->local_calls[i]; - int32_t target_loc; - assert(local_call.target_offset == 0); - assert(local_call.target_pc != TARGET_PC_EXIT); - assert(local_call.target_pc != TARGET_PC_RETPOLINE); - target_loc = state->pc_locs[local_call.target_pc]; + int32_t target_loc = 0; + // A local call must be eBPF PC-relative and it cannot be special. + assert(!local_call.target.is_special); + target_loc = state->pc_locs[local_call.target.target.regular.ebpf_target_pc]; int32_t rel = target_loc - local_call.offset_loc; rel -= state->bpf_function_prolog_size; diff --git a/vm/ubpf_jit_support.c b/vm/ubpf_jit_support.c index 37ec692d2..38ce665fe 100644 --- a/vm/ubpf_jit_support.c +++ b/vm/ubpf_jit_support.c @@ -21,6 +21,7 @@ #include #include "ubpf.h" #include "ubpf_int.h" +#include int initialize_jit_state_result( @@ -77,46 +78,32 @@ release_jit_state_result(struct jit_state* state, struct ubpf_jit_result* compil } void -emit_patchable_relative_ex( - uint32_t offset, - uint32_t target_pc, - uint32_t manual_target_offset, - struct patchable_relative* table, - size_t index, - bool near) +emit_patchable_relative(struct patchable_relative* table, + uint32_t offset, struct PatchableTarget target, size_t index) { struct patchable_relative* jump = &table[index]; jump->offset_loc = offset; - jump->target_pc = target_pc; - jump->target_offset = manual_target_offset; - jump->near = near; + jump->target = target; } void -emit_patchable_relative( - uint32_t offset, uint32_t target_pc, uint32_t manual_target_offset, struct patchable_relative* table, size_t index) +note_load(struct jit_state* state, struct PatchableTarget target) { - emit_patchable_relative_ex(offset, target_pc, manual_target_offset, table, index, false); + emit_patchable_relative(state->loads, state->offset, target, state->num_loads++); } void -note_load(struct jit_state* state, uint32_t target_pc) +note_lea(struct jit_state* state, struct PatchableTarget target) { - emit_patchable_relative(state->offset, target_pc, 0, state->loads, state->num_loads++); + emit_patchable_relative(state->leas, state->offset, target, state->num_leas++); } void -note_lea(struct jit_state* state, uint32_t offset) -{ - emit_patchable_relative(state->offset, offset, 0, state->leas, state->num_leas++); -} - -void -fixup_jump_target(struct patchable_relative* table, size_t table_size, uint32_t src_offset, uint32_t dest_offset) +modify_patchable_relatives_target(struct patchable_relative* table, size_t table_size, uint32_t patchable_relative_src, struct PatchableTarget target) { for (size_t index = 0; index < table_size; index++) { - if (table[index].offset_loc == src_offset) { - table[index].target_offset = dest_offset; + if (table[index].offset_loc == patchable_relative_src) { + table[index].target = target; } } } @@ -124,5 +111,9 @@ fixup_jump_target(struct patchable_relative* table, size_t table_size, uint32_t void emit_jump_target(struct jit_state* state, uint32_t jump_src) { - fixup_jump_target(state->jumps, state->num_jumps, jump_src, state->offset); + DECLARE_PATCHABLE_TARGET(pt); + pt.is_special = false; + pt.target.regular.jit_target_pc = state->offset; + + modify_patchable_relatives_target(state->jumps, state->num_jumps, jump_src, pt); } diff --git a/vm/ubpf_jit_support.h b/vm/ubpf_jit_support.h index 66f67ea8b..1f80a2da4 100644 --- a/vm/ubpf_jit_support.h +++ b/vm/ubpf_jit_support.h @@ -40,25 +40,81 @@ enum JitProgress UnknownInstruction }; -struct patchable_relative + +/* + * During the process of JITing, the targets of program-control + * instructions are not always known. The functions of below + * make it possible to emit program-control instructions with + * temporary targets and then fixup those instructions when their + * targets are known. Some of the targets are _special_ (SpecialTarget) + * and others are _regular_ (RegularTarget). No matter what, however, + * during JITing, the targets of program-control instructions are + * PatchableTargets. + */ +enum SpecialTarget { - /* Where in the instruction stream should this relative address be patched. */ - uint32_t offset_loc; - /* Which PC should this target. The ultimate offset will be determined + Exit, + Enter, + Retpoline, + ExternalDispatcher, + LoadHelperTable, +}; + +struct RegularTarget +{ + /* Which eBPF PC should this target. The ultimate offset will be determined * automatically unless ... */ - uint32_t target_pc; - /* ... the target_offset is set which overrides the automatic lookup. */ - uint32_t target_offset; - /* Whether or not this patchable relative is _near_. */ + uint32_t ebpf_target_pc; + /* ... the JIT target_offset is set which overrides the automatic lookup. */ + uint32_t jit_target_pc; + /* Whether or not this target is _near_ the source. */ bool near; }; -/* Special values for target_pc in struct jump */ -#define TARGET_PC_EXIT ~UINT32_C(0) -#define TARGET_PC_ENTER (~UINT32_C(0) & 0x01) -#define TARGET_PC_RETPOLINE (~UINT32_C(0) & 0x0101) -#define TARGET_PC_EXTERNAL_DISPATCHER (~UINT32_C(0) & 0x010101) -#define TARGET_LOAD_HELPER_TABLE (~UINT32_C(0) & 0x01010101) +struct PatchableTarget +{ + bool is_special; + union + { + enum SpecialTarget special; + struct RegularTarget regular; + } target; +}; + +#define DECLARE_PATCHABLE_TARGET(x) \ + struct PatchableTarget x; \ + memset(&x, 0, sizeof(struct PatchableTarget)); + +#define DECLARE_PATCHABLE_SPECIAL_TARGET(x, tgt) \ + struct PatchableTarget x; \ + memset(&x, 0, sizeof(struct PatchableTarget)); \ + x.is_special = true; \ + x.target.special = tgt; \ + +#define DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(x, tgt) \ + struct PatchableTarget x; \ + memset(&x, 0, sizeof(struct PatchableTarget)); \ + x.is_special = false; \ + x.target.regular.ebpf_target_pc = tgt; + +#define DECLARE_PATCHABLE_REGULAR_JIT_TARGET(x, tgt) \ + struct PatchableTarget x; \ + memset(&x, 0, sizeof(struct PatchableTarget)); \ + x.is_special = false; \ + x.target.regular.jit_target_pc = tgt; + + +struct patchable_relative +{ + /* Where in the JIT'd instruction stream should the actual + * target be written once it is determined. + */ + uint32_t offset_loc; + + /* How to calculate the actual target. + */ + struct PatchableTarget target; +}; struct jit_state { @@ -119,47 +175,27 @@ release_jit_state_result(struct jit_state* state, struct ubpf_jit_result* compil * Emitting an entry into the patchable relative table means that resolution of the target * address can be postponed until all the instructions are emitted. Note: This function does * not emit any instructions -- it simply updates metadata to guide resolution after code generation. - * _target_pc_ is in eBPF instruction units and _manual_target_offset_ is in JIT'd instruction - * units. In other words, setting _target_pc_ instead of _manual_target_offset_ will guide - * the resolution algorithm to find the JIT'd code that corresponds to the eBPF instruction - * (as the jump target); alternatively, setting _manual_target_offset_ will direct the - * resolution algorithm to find the JIT'd instruction at that offset (as the target). - * - * @param[in] offset The offset in the JIT'd code where the to-be-resolved target begins. - * @param[in] target_pc The offset of the eBPF instruction targeted by the jump. - * @param[in] manual_target_offset The offset of the JIT'd instruction targeted by the jump. - * A non-zero value for this parameter overrides _target_pc_`. + * * @param[in] table The relative patchable table to update. + * @param[in] offset The offset in the JIT'd code where the to-be-resolved target begins. * @param[in] index A spot in the _table_ to add/update according to the given parameters. - * @param[in] near Whether the target is relatively near the jump. - */ -void -emit_patchable_relative_ex( - uint32_t offset, - uint32_t target_pc, - uint32_t manual_target_offset, - struct patchable_relative* table, - size_t index, - bool near); - -/** @brief Add an entry to the given patchable relative table. - * - * See emit_patchable_relative_ex. emit_patchable_relative's parameters have the same meaning - * but fixes the _near_ argument to false. */ void -emit_patchable_relative( - uint32_t offset, uint32_t target_pc, uint32_t manual_target_offset, struct patchable_relative* table, size_t index); +emit_patchable_relative(struct patchable_relative* table, uint32_t offset, struct PatchableTarget target, size_t index); void -note_load(struct jit_state* state, uint32_t target_pc); +note_load(struct jit_state* state, struct PatchableTarget target); void -note_lea(struct jit_state* state, uint32_t offset); +note_lea(struct jit_state* state, struct PatchableTarget target); void emit_jump_target(struct jit_state* state, uint32_t jump_src); void -fixup_jump_target(struct patchable_relative* table, size_t table_size, uint32_t src_offset, uint32_t dest_offset); +modify_patchable_relatives_target( + struct patchable_relative* table, + size_t table_size, + uint32_t src_offset, + struct PatchableTarget target); #endif diff --git a/vm/ubpf_jit_x86_64.c b/vm/ubpf_jit_x86_64.c index 97f7aa53e..381288dc9 100644 --- a/vm/ubpf_jit_x86_64.c +++ b/vm/ubpf_jit_x86_64.c @@ -187,40 +187,27 @@ emit_4byte_offset_placeholder(struct jit_state* state) } static uint32_t -emit_jump_address_reloc(struct jit_state* state, int32_t target_pc) +emit_jump_address_reloc(struct jit_state* state, struct PatchableTarget target) { if (state->num_jumps == UBPF_MAX_INSTS) { state->jit_status = TooManyJumps; return 0; } uint32_t target_address_offset = state->offset; - emit_patchable_relative(state->offset, target_pc, 0, state->jumps, state->num_jumps++); + emit_patchable_relative(state->jumps, state->offset, target, state->num_jumps++); emit_4byte_offset_placeholder(state); return target_address_offset; } static uint32_t -emit_near_jump_address_reloc(struct jit_state* state, int32_t target_pc) -{ - if (state->num_jumps == UBPF_MAX_INSTS) { - state->jit_status = TooManyJumps; - return 0; - } - uint32_t target_address_offset = state->offset; - emit_patchable_relative_ex(state->offset, target_pc, 0, state->jumps, state->num_jumps++, true /* near */); - emit1(state, 0x0); - return target_address_offset; -} - -static uint32_t -emit_local_call_address_reloc(struct jit_state* state, int32_t target_pc) +emit_local_call_address_reloc(struct jit_state* state, struct PatchableTarget target) { if (state->num_local_calls == UBPF_MAX_INSTS) { state->jit_status = TooManyLocalCalls; return 0; } uint32_t target_address_offset = state->offset; - emit_patchable_relative(state->offset, target_pc, 0, state->local_calls, state->num_local_calls++); + emit_patchable_relative(state->local_calls, state->offset, target, state->num_local_calls++); emit_4byte_offset_placeholder(state); return target_address_offset; } @@ -410,11 +397,11 @@ emit_cmp32(struct jit_state* state, int src, int dst) } static inline uint32_t -emit_jcc(struct jit_state* state, int code, int32_t target_pc) +emit_jcc(struct jit_state* state, int code, struct PatchableTarget target) { emit1(state, 0x0f); emit1(state, code); - return emit_jump_address_reloc(state, target_pc); + return emit_jump_address_reloc(state, target); } /* Load [src + offset] into dst */ @@ -450,7 +437,7 @@ emit_load_imm(struct jit_state* state, int dst, int64_t imm) } static uint32_t -emit_rip_relative_load(struct jit_state* state, int dst, int relative_load_tgt) +emit_rip_relative_load(struct jit_state* state, int dst, struct PatchableTarget load_tgt) { if (state->num_loads == UBPF_MAX_INSTS) { state->jit_status = TooManyLoads; @@ -460,14 +447,16 @@ emit_rip_relative_load(struct jit_state* state, int dst, int relative_load_tgt) emit_rex(state, 1, 0, 0, 0); emit1(state, 0x8b); emit_modrm(state, 0, dst, 0x05); + uint32_t load_target_offset = state->offset; - note_load(state, relative_load_tgt); + note_load(state, load_tgt); emit_4byte_offset_placeholder(state); + return load_target_offset; } static void -emit_rip_relative_lea(struct jit_state* state, int dst, int lea_tgt) +emit_rip_relative_lea(struct jit_state* state, int lea_dst_reg, struct PatchableTarget lea_tgt) { if (state->num_leas == UBPF_MAX_INSTS) { state->jit_status = TooManyLeas; @@ -477,7 +466,7 @@ emit_rip_relative_lea(struct jit_state* state, int dst, int lea_tgt) // lea dst, [rip + HELPER TABLE ADDRESS] emit_rex(state, 1, 1, 0, 0); emit1(state, 0x8d); - emit_modrm(state, 0, dst, 0x05); + emit_modrm(state, 0, lea_dst_reg, 0x05); note_lea(state, lea_tgt); emit_4byte_offset_placeholder(state); } @@ -530,32 +519,22 @@ emit_ret(struct jit_state* state) * @return The offset in the JIT'd code where the jump offset starts. */ static inline uint32_t -emit_jmp(struct jit_state* state, uint32_t target_pc) +emit_jmp(struct jit_state* state, struct PatchableTarget target) { + if (!target.is_special && target.target.regular.near) { + emit1(state, 0xeb); + return emit_jump_address_reloc(state, target); + } emit1(state, 0xe9); - return emit_jump_address_reloc(state, target_pc); + return emit_jump_address_reloc(state, target); } -/** @brief Emit a near jump. - * - * @param[in] state The JIT state. - * @param[in] target_pc The PC to which to jump when this near - * jump is executed. - * @return The offset in the JIT'd code where the jump offset starts. - */ static inline uint32_t -emit_near_jmp(struct jit_state* state, uint32_t target_pc) -{ - emit1(state, 0xeb); - return emit_near_jump_address_reloc(state, target_pc); -} - -static inline uint32_t -emit_call(struct jit_state* state, uint32_t target_pc) +emit_call(struct jit_state* state, struct PatchableTarget target) { emit1(state, 0xe8); uint32_t call_src = state->offset; - emit_jump_address_reloc(state, target_pc); + emit_jump_address_reloc(state, target); return call_src; } @@ -622,11 +601,18 @@ emit_dispatched_external_helper_call(struct jit_state* state, unsigned int idx) emit_alu64_imm32(state, 0x81, 5, RSP, 3 * sizeof(uint64_t)); #endif - emit_rip_relative_load(state, RAX, TARGET_PC_EXTERNAL_DISPATCHER); + DECLARE_PATCHABLE_TARGET(rip_rel_tgt); + rip_rel_tgt.is_special = true; + rip_rel_tgt.target.special = ExternalDispatcher; + emit_rip_relative_load(state, RAX, rip_rel_tgt); + // cmp rax, 0 emit_cmp_imm32(state, RAX, 0); // jne skip_default_dispatcher_label - uint32_t skip_default_dispatcher_source = emit_jcc(state, 0x85, 0); + DECLARE_PATCHABLE_TARGET(default_jmp_tgt); + default_jmp_tgt.is_special = false; + default_jmp_tgt.target.regular.ebpf_target_pc = 0; + uint32_t skip_default_dispatcher_source = emit_jcc(state, 0x85, default_jmp_tgt); // Default dispatcher: @@ -638,7 +624,11 @@ emit_dispatched_external_helper_call(struct jit_state* state, unsigned int idx) emit_alu64_imm8(state, 0xc1, 4, RAX, 3); // lea r10, [rip + HELPER TABLE ADDRESS] - emit_rip_relative_lea(state, R10, TARGET_LOAD_HELPER_TABLE); + DECLARE_PATCHABLE_TARGET(rip_rel_load_helper_tgt); + rip_rel_load_helper_tgt.is_special = true; + rip_rel_load_helper_tgt.target.special = LoadHelperTable; + // emit_rip_relative_lea(state, R10, TARGET_LOAD_HELPER_TABLE); + emit_rip_relative_lea(state, R10, rip_rel_load_helper_tgt); // add rax, r10 emit_alu64(state, 0x01, R10, RAX); @@ -661,7 +651,7 @@ emit_dispatched_external_helper_call(struct jit_state* state, unsigned int idx) #endif // jmp call_label - uint32_t skip_external_dispatcher_source = emit_jmp(state, 0); + uint32_t skip_external_dispatcher_source = emit_jmp(state, default_jmp_tgt); // External dispatcher: @@ -717,7 +707,11 @@ emit_dispatched_external_helper_call(struct jit_state* state, unsigned int idx) #endif #ifndef UBPF_DISABLE_RETPOLINES - emit_call(state, TARGET_PC_RETPOLINE); + DECLARE_PATCHABLE_TARGET(retpoline_tgt); + retpoline_tgt.is_special = true; + retpoline_tgt.target.special = Retpoline; + // emit_call(state, TARGET_PC_RETPOLINE); + emit_call(state, retpoline_tgt); #else /* TODO use direct call when possible */ /* callq *%rax */ @@ -1127,7 +1121,7 @@ emit_muldivmod(struct jit_state* state, uint8_t opcode, int src, int dst, int32_ } } static inline void -emit_local_call(struct ubpf_vm* vm, struct jit_state* state, uint32_t target_pc) +emit_local_call(struct ubpf_vm* vm, struct jit_state* state, uint32_t ebpf_target_pc) { UNUSED_PARAMETER(vm); // Invariant: The top of the host stack always holds the amount of space needed @@ -1152,7 +1146,8 @@ emit_local_call(struct ubpf_vm* vm, struct jit_state* state, uint32_t target_pc) emit_alu64_imm32(state, 0x81, 5, RSP, 4 * sizeof(uint64_t)); #endif emit1(state, 0xe8); // e8 is the opcode for a CALL - emit_local_call_address_reloc(state, target_pc); + DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(target_pt, ebpf_target_pc); + emit_local_call_address_reloc(state, target_pt); #if defined(_WIN32) /* Deallocate home register space - 4 registers */ @@ -1204,14 +1199,16 @@ emit_retpoline(struct jit_state* state) /* label0: */ /* call label1 */ uint32_t retpoline_target = state->offset; - uint32_t label1_call_offset = emit_call(state, 0); + DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(default_call_tgt, 0); + uint32_t label1_call_offset = emit_call(state, default_call_tgt); /* capture_ret_spec: */ /* pause */ uint32_t capture_ret_spec = state->offset; emit_pause(state); /* jmp capture_ret_spec */ - emit_jmp(state, capture_ret_spec); + DECLARE_PATCHABLE_REGULAR_JIT_TARGET(capture_ret_tgt, capture_ret_spec); + emit_jmp(state, capture_ret_tgt); /* label1: */ /* mov rax, (rsp) */ @@ -1224,7 +1221,8 @@ emit_retpoline(struct jit_state* state) /* ret */ emit_ret(state); - fixup_jump_target(state->jumps, state->num_jumps, label1_call_offset, label1); + DECLARE_PATCHABLE_REGULAR_JIT_TARGET(label1_tgt, label1); + modify_patchable_relatives_target(state->jumps, state->num_jumps, label1_call_offset, label1_tgt); return retpoline_target; } @@ -1354,7 +1352,9 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) * We jump over this instruction in the first place; return here * after the eBPF program is finished executing. */ - emit_jmp(state, TARGET_PC_EXIT); + + DECLARE_PATCHABLE_SPECIAL_TARGET(exit_tgt, Exit) + emit_jmp(state, exit_tgt); for (i = 0; i < vm->num_insts; i++) { if (state->jit_status != NoError) { @@ -1367,6 +1367,8 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) int src = map_register(inst.src); uint32_t target_pc = i + inst.offset + 1; + DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(tgt, target_pc); + // If // a) the previous instruction in the eBPF program could fallthrough // to this instruction and @@ -1379,7 +1381,9 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) if (i != 0 && vm->int_funcs[i]) { struct ebpf_inst prev_inst = ubpf_fetch_instruction(vm, i - 1); if (ubpf_instruction_has_fallthrough(prev_inst)) { - fallthrough_jump_source = emit_near_jmp(state, 0); + DECLARE_PATCHABLE_REGULAR_EBPF_TARGET(default_near_target, 0) + default_near_target.target.regular.near = true; + fallthrough_jump_source = emit_jmp(state, default_near_target); fallthrough_jump_present = true; } } @@ -1416,7 +1420,10 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) // If there was a jump inserted to bypass the host stack manipulation code, // we need to update its target. if (fallthrough_jump_present) { - fixup_jump_target(state->jumps, state->num_jumps, fallthrough_jump_source, state->offset); + DECLARE_PATCHABLE_REGULAR_JIT_TARGET(fallthrough_jump_tgt, state->offset); + fallthrough_jump_tgt.target.regular.near = true; + modify_patchable_relatives_target( + state->jumps, state->num_jumps, fallthrough_jump_source, fallthrough_jump_tgt); } state->pc_locs[i] = state->offset; @@ -1586,183 +1593,183 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) /* TODO use 8 bit immediate when possible */ case EBPF_OP_JA: - emit_jmp(state, target_pc); + emit_jmp(state, tgt); break; case EBPF_OP_JEQ_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x84, target_pc); + emit_jcc(state, 0x84, tgt); break; case EBPF_OP_JEQ_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x84, target_pc); + emit_jcc(state, 0x84, tgt); break; case EBPF_OP_JGT_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x87, target_pc); + emit_jcc(state, 0x87, tgt); break; case EBPF_OP_JGT_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x87, target_pc); + emit_jcc(state, 0x87, tgt); break; case EBPF_OP_JGE_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x83, target_pc); + emit_jcc(state, 0x83, tgt); break; case EBPF_OP_JGE_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x83, target_pc); + emit_jcc(state, 0x83, tgt); break; case EBPF_OP_JLT_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x82, target_pc); + emit_jcc(state, 0x82, tgt); break; case EBPF_OP_JLT_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x82, target_pc); + emit_jcc(state, 0x82, tgt); break; case EBPF_OP_JLE_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x86, target_pc); + emit_jcc(state, 0x86, tgt); break; case EBPF_OP_JLE_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x86, target_pc); + emit_jcc(state, 0x86, tgt); break; case EBPF_OP_JSET_IMM: emit_alu64_imm32(state, 0xf7, 0, dst, inst.imm); - emit_jcc(state, 0x85, target_pc); + emit_jcc(state, 0x85, tgt); break; case EBPF_OP_JSET_REG: emit_alu64(state, 0x85, src, dst); - emit_jcc(state, 0x85, target_pc); + emit_jcc(state, 0x85, tgt); break; case EBPF_OP_JNE_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x85, target_pc); + emit_jcc(state, 0x85, tgt); break; case EBPF_OP_JNE_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x85, target_pc); + emit_jcc(state, 0x85, tgt); break; case EBPF_OP_JSGT_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x8f, target_pc); + emit_jcc(state, 0x8f, tgt); break; case EBPF_OP_JSGT_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x8f, target_pc); + emit_jcc(state, 0x8f, tgt); break; case EBPF_OP_JSGE_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x8d, target_pc); + emit_jcc(state, 0x8d, tgt); break; case EBPF_OP_JSGE_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x8d, target_pc); + emit_jcc(state, 0x8d, tgt); break; case EBPF_OP_JSLT_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x8c, target_pc); + emit_jcc(state, 0x8c, tgt); break; case EBPF_OP_JSLT_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x8c, target_pc); + emit_jcc(state, 0x8c, tgt); break; case EBPF_OP_JSLE_IMM: emit_cmp_imm32(state, dst, inst.imm); - emit_jcc(state, 0x8e, target_pc); + emit_jcc(state, 0x8e, tgt); break; case EBPF_OP_JSLE_REG: emit_cmp(state, src, dst); - emit_jcc(state, 0x8e, target_pc); + emit_jcc(state, 0x8e, tgt); break; case EBPF_OP_JEQ32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x84, target_pc); + emit_jcc(state, 0x84, tgt); break; case EBPF_OP_JEQ32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x84, target_pc); + emit_jcc(state, 0x84, tgt); break; case EBPF_OP_JGT32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x87, target_pc); + emit_jcc(state, 0x87, tgt); break; case EBPF_OP_JGT32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x87, target_pc); + emit_jcc(state, 0x87, tgt); break; case EBPF_OP_JGE32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x83, target_pc); + emit_jcc(state, 0x83, tgt); break; case EBPF_OP_JGE32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x83, target_pc); + emit_jcc(state, 0x83, tgt); break; case EBPF_OP_JLT32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x82, target_pc); + emit_jcc(state, 0x82, tgt); break; case EBPF_OP_JLT32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x82, target_pc); + emit_jcc(state, 0x82, tgt); break; case EBPF_OP_JLE32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x86, target_pc); + emit_jcc(state, 0x86, tgt); break; case EBPF_OP_JLE32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x86, target_pc); + emit_jcc(state, 0x86, tgt); break; case EBPF_OP_JSET32_IMM: emit_alu32_imm32(state, 0xf7, 0, dst, inst.imm); - emit_jcc(state, 0x85, target_pc); + emit_jcc(state, 0x85, tgt); break; case EBPF_OP_JSET32_REG: emit_alu32(state, 0x85, src, dst); - emit_jcc(state, 0x85, target_pc); + emit_jcc(state, 0x85, tgt); break; case EBPF_OP_JNE32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x85, target_pc); + emit_jcc(state, 0x85, tgt); break; case EBPF_OP_JNE32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x85, target_pc); + emit_jcc(state, 0x85, tgt); break; case EBPF_OP_JSGT32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x8f, target_pc); + emit_jcc(state, 0x8f, tgt); break; case EBPF_OP_JSGT32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x8f, target_pc); + emit_jcc(state, 0x8f, tgt); break; case EBPF_OP_JSGE32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x8d, target_pc); + emit_jcc(state, 0x8d, tgt); break; case EBPF_OP_JSGE32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x8d, target_pc); + emit_jcc(state, 0x8d, tgt); break; case EBPF_OP_JSLT32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x8c, target_pc); + emit_jcc(state, 0x8c, tgt); break; case EBPF_OP_JSLT32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x8c, target_pc); + emit_jcc(state, 0x8c, tgt); break; case EBPF_OP_JSLE32_IMM: emit_cmp32_imm32(state, dst, inst.imm); - emit_jcc(state, 0x8e, target_pc); + emit_jcc(state, 0x8e, tgt); break; case EBPF_OP_JSLE32_REG: emit_cmp32(state, src, dst); - emit_jcc(state, 0x8e, target_pc); + emit_jcc(state, 0x8e, tgt); break; case EBPF_OP_CALL: /* We reserve RCX for shifts */ @@ -1771,7 +1778,10 @@ translate(struct ubpf_vm* vm, struct jit_state* state, char** errmsg) emit_dispatched_external_helper_call(state, inst.imm); if (inst.imm == vm->unwind_stack_extension_index) { emit_cmp_imm32(state, map_register(BPF_REG_0), 0); - emit_jcc(state, 0x84, TARGET_PC_EXIT); + DECLARE_PATCHABLE_TARGET(exit_tgt); + exit_tgt.is_special = true; + exit_tgt.target.special = Exit; + emit_jcc(state, 0x84, exit_tgt); } } else if (inst.src == 1) { target_pc = i + inst.imm + 1; @@ -2009,17 +2019,30 @@ resolve_patchable_relatives(struct jit_state* state) struct patchable_relative jump = state->jumps[i]; int target_loc; - if (jump.target_offset != 0) { - target_loc = jump.target_offset; - } else if (jump.target_pc == TARGET_PC_EXIT) { - target_loc = state->exit_loc; - } else if (jump.target_pc == TARGET_PC_RETPOLINE) { - target_loc = state->retpoline_loc; + bool is_near = false; + + if (jump.target.is_special) { + // There are only two special targets for jumps: Exit and Retpoline. + if (jump.target.target.special == Exit) { + target_loc = state->exit_loc; + } else if (jump.target.target.special == Retpoline) { + target_loc = state->retpoline_loc; + } else { + target_loc = -1; + return false; + } } else { - target_loc = state->pc_locs[jump.target_pc]; + // The jit target, if specified, takes precedence. + if (jump.target.target.regular.jit_target_pc != 0) { + target_loc = jump.target.target.regular.jit_target_pc; + } else { + target_loc = state->pc_locs[jump.target.target.regular.ebpf_target_pc]; + } + + is_near = jump.target.target.regular.near; } - if (jump.near) { + if (is_near) { /* When there is a near jump, we need to make sure that the target * is within the proper limits. So, we start with a type that can * hold values that are bigger than we'll ultimately need. If we @@ -2047,12 +2070,12 @@ resolve_patchable_relatives(struct jit_state* state) for (i = 0; i < state->num_local_calls; i++) { struct patchable_relative local_call = state->local_calls[i]; - int target_loc; - assert(local_call.target_offset == 0); - assert(local_call.target_pc != TARGET_PC_EXIT); - assert(local_call.target_pc != TARGET_PC_RETPOLINE); + // There are no special local calls and + assert(!local_call.target.is_special); + // the targets must all be eBPF-relative PCs. + assert(local_call.target.target.regular.jit_target_pc == 0); - target_loc = state->pc_locs[local_call.target_pc]; + int target_loc = state->pc_locs[local_call.target.target.regular.ebpf_target_pc]; /* Assumes call offset is at end of instruction */ uint32_t rel = target_loc - (local_call.offset_loc + sizeof(uint32_t)); @@ -2065,9 +2088,9 @@ resolve_patchable_relatives(struct jit_state* state) for (i = 0; i < state->num_loads; i++) { struct patchable_relative load = state->loads[i]; - int target_loc; + int target_loc = 0; // It is only possible to load from the external dispatcher's position. - if (load.target_pc == TARGET_PC_EXTERNAL_DISPATCHER) { + if (load.target.is_special && load.target.target.special == ExternalDispatcher) { target_loc = state->dispatcher_loc; } else { target_loc = -1; @@ -2085,7 +2108,7 @@ resolve_patchable_relatives(struct jit_state* state) int target_loc; // It is only possible to LEA from the helper table. - if (lea.target_pc == TARGET_LOAD_HELPER_TABLE) { + if (lea.target.is_special && lea.target.target.special == LoadHelperTable) { target_loc = state->helper_table_loc; } else { target_loc = -1;