24uint32_t sig_imm_size(int64_t imm)
27 if (imm >= INT8_MIN && imm <= INT8_MAX)
29 if (imm >= INT16_MIN && imm <= INT16_MAX)
31 if (imm >= INT32_MIN && imm <= INT32_MAX)
38uint32_t unsig_imm_size(uint64_t imm)
43 else if (imm <= UINT16_MAX)
45 else if (imm <= UINT32_MAX)
53 bool is_iprel = base_reg.as.reg.reg_type == REG_IP;
58 .as.mem = { base_reg.as.reg.reg_no, 0, 0,
false, is_iprel, disp }
81 rb_bug(
"yjit: scale not one of 1,2,4,8");
85 bool is_iprel = base_reg.as.reg.reg_type == REG_IP;
91 .base_reg_no = base_reg.as.reg.reg_no,
92 .idx_reg_no = index_reg.as.reg.reg_no,
94 .scale_exp = scale_exp,
105 assert (num_bits % 8 == 0);
107 sub.num_bits = num_bits;
127 .as.unsig_imm = (uint64_t)ptr
134static uint8_t *align_ptr(uint8_t *ptr, uint32_t multiple)
137 uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
144 uint32_t pad = multiple - rem;
150static uint8_t *alloc_exec_mem(uint32_t mem_size)
156 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
158 uint32_t page_size = (uint32_t)sysconf(_SC_PAGESIZE);
159 uint8_t *req_addr = align_ptr((uint8_t*)&alloc_exec_mem, page_size);
163 mem_block = (uint8_t*)mmap(
166 PROT_READ | PROT_EXEC,
167 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
173 if (mem_block != MAP_FAILED) {
178 req_addr += 4 * 1024 * 1024;
179 }
while (req_addr < (uint8_t*)&alloc_exec_mem + INT32_MAX);
184 mem_block = (uint8_t*)mmap(
185 (
void*)alloc_exec_mem,
187 PROT_READ | PROT_EXEC,
188 MAP_PRIVATE | MAP_ANONYMOUS,
195 if (mem_block == MAP_FAILED) {
197 mem_block = (uint8_t*)mmap(
200 PROT_READ | PROT_EXEC,
201 MAP_PRIVATE | MAP_ANONYMOUS,
208 if (mem_block == MAP_FAILED) {
209 perror(
"mmap call failed");
216 cb_init(cb, mem_block, mem_size);
221 cb_mark_all_writeable(cb);
222 memset(mem_block, 0x1E, mem_size);
223 cb_mark_all_executable(cb);
233void cb_init(
codeblock_t *cb, uint8_t *mem_block, uint32_t mem_size)
236 cb->mem_block_ = mem_block;
237 cb->mem_size = mem_size;
241 cb->current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
250 assert (pos < cb->mem_size);
255void cb_align_pos(
codeblock_t *cb, uint32_t multiple)
258 uint8_t *ptr = cb_get_write_ptr(cb);
259 uint8_t *aligned_ptr = align_ptr(ptr, multiple);
260 const uint32_t write_pos = cb->write_pos;
263 ptrdiff_t pad = aligned_ptr - ptr;
264 cb_set_pos(cb, write_pos + (int32_t)pad);
268void cb_set_write_ptr(
codeblock_t *cb, uint8_t *code_ptr)
270 intptr_t pos = code_ptr - cb->mem_block_;
271 assert (pos < cb->mem_size);
272 cb_set_pos(cb, (uint32_t)pos);
276uint8_t *cb_get_ptr(
const codeblock_t *cb, uint32_t index)
278 if (index < cb->mem_size) {
279 return &cb->mem_block_[index];
289 return cb_get_ptr(cb, cb->write_pos);
295 assert (cb->mem_block_);
296 if (cb->write_pos < cb->mem_size) {
297 cb_mark_position_writeable(cb, cb->write_pos);
298 cb->mem_block_[cb->write_pos] = byte;
302 cb->dropped_bytes =
true;
307void cb_write_bytes(
codeblock_t *cb, uint32_t num_bytes, ...)
310 va_start(va, num_bytes);
312 for (uint32_t i = 0; i < num_bytes; ++i)
314 uint8_t
byte = va_arg(va,
int);
315 cb_write_byte(cb,
byte);
322void cb_write_int(
codeblock_t *cb, uint64_t val, uint32_t num_bits)
324 assert (num_bits > 0);
325 assert (num_bits % 8 == 0);
330 cb_write_byte(cb, (uint8_t)val);
337 (uint8_t)((val >> 0) & 0xFF),
338 (uint8_t)((val >> 8) & 0xFF)
346 (uint8_t)((val >> 0) & 0xFF),
347 (uint8_t)((val >> 8) & 0xFF),
348 (uint8_t)((val >> 16) & 0xFF),
349 (uint8_t)((val >> 24) & 0xFF)
356 uint32_t num_bytes = num_bits / 8;
359 for (uint32_t i = 0; i < num_bytes; ++i)
361 uint8_t byte_val = (uint8_t)(val & 0xFF);
362 cb_write_byte(cb, byte_val);
370uint32_t cb_new_label(
codeblock_t *cb,
const char *name)
375 assert (cb->num_labels < MAX_LABELS);
378 uint32_t label_idx = cb->num_labels++;
381 cb->label_addrs[label_idx] = 0;
382 cb->label_names[label_idx] = name;
388void cb_write_label(
codeblock_t *cb, uint32_t label_idx)
390 assert (label_idx < MAX_LABELS);
391 cb->label_addrs[label_idx] = cb->write_pos;
395void cb_label_ref(
codeblock_t *cb, uint32_t label_idx)
397 assert (label_idx < MAX_LABELS);
398 assert (cb->num_refs < MAX_LABEL_REFS);
401 cb->label_refs[cb->num_refs] = (
labelref_t){ cb->write_pos, label_idx };
408 uint32_t orig_pos = cb->write_pos;
411 for (uint32_t i = 0; i < cb->num_refs; ++i)
413 uint32_t ref_pos = cb->label_refs[i].pos;
414 uint32_t label_idx = cb->label_refs[i].label_idx;
415 assert (ref_pos < cb->mem_size);
416 assert (label_idx < MAX_LABELS);
418 uint32_t label_addr = cb->label_addrs[label_idx];
419 assert (label_addr < cb->mem_size);
422 int64_t offset = (int64_t)label_addr - (int64_t)(ref_pos + 4);
424 cb_set_pos(cb, ref_pos);
425 cb_write_int(cb, offset, 32);
428 cb->write_pos = orig_pos;
438 if (opnd.type == OPND_NONE || opnd.type == OPND_IMM)
443 if (opnd.type == OPND_REG)
446 opnd.as.reg.reg_no > 7 ||
447 (opnd.num_bits == 8 && opnd.as.reg.reg_no >= 4 && opnd.as.reg.reg_no <= 7)
451 if (opnd.type == OPND_MEM)
462 if (opnd.type != OPND_MEM)
475 assert (opnd.type == OPND_MEM);
484 if (opnd.as.mem.
disp != 0)
486 uint32_t num_bits = sig_imm_size(opnd.as.mem.
disp);
487 assert (num_bits <= 32 &&
"displacement does not fit in 32 bits");
497 if (opnd.as.mem.
base_reg_no == RBP.as.reg.reg_no ||
507static void cb_write_rex(
520 uint8_t w = w_flag? 1:0;
521 uint8_t r = (reg_no & 8)? 1:0;
522 uint8_t x = (idx_reg_no & 8)? 1:0;
523 uint8_t b = (rm_reg_no & 8)? 1:0;
526 uint8_t rexByte = 0x40 + (w << 3) + (r << 2) + (x << 1) + (b);
527 cb_write_byte(cb, rexByte);
534 uint8_t op_byte = opcode | (reg.as.reg.reg_no & 7);
535 cb_write_byte(cb, op_byte);
539static void cb_write_rm(
549 assert (op_len > 0 && op_len <= 3);
550 assert (r_opnd.type == OPND_REG || r_opnd.type == OPND_NONE);
553 bool need_rex = rexW || rex_needed(r_opnd) || rex_needed(rm_opnd);
556 bool need_sib = sib_needed(r_opnd) || sib_needed(rm_opnd);
560 cb_write_byte(cb, 0x66);
571 uint8_t w = rexW? 1:0;
574 if (r_opnd.type != OPND_NONE)
575 r = (r_opnd.as.reg.reg_no & 8)? 1:0;
580 if (need_sib && rm_opnd.as.mem.
has_idx)
586 if (rm_opnd.type == OPND_REG)
587 b = (rm_opnd.as.reg.reg_no & 8)? 1:0;
588 else if (rm_opnd.type == OPND_MEM)
594 uint8_t rex_byte = 0x40 + (w << 3) + (r << 2) + (x << 1) + (b);
595 cb_write_byte(cb, rex_byte);
600 va_start(va, op_len);
601 for (uint32_t i = 0; i < op_len; ++i)
603 uint8_t
byte = va_arg(va,
int);
604 cb_write_byte(cb,
byte);
613 !(opExt != 0xFF && r_opnd.type != OPND_NONE) &&
614 "opcode extension and register operand present"
619 if (rm_opnd.type == OPND_REG)
625 uint32_t dsize = disp_size(rm_opnd);
626 if (dsize == 0 || rm_opnd.as.mem.
is_iprel)
630 else if (dsize == 32)
640 else if (r_opnd.type == OPND_REG)
641 reg = r_opnd.as.reg.reg_no & 7;
647 if (rm_opnd.type == OPND_REG)
649 rm = rm_opnd.as.reg.reg_no & 7;
660 uint8_t rm_byte = (mod << 6) + (reg << 3) + (rm);
661 cb_write_byte(cb, rm_byte);
670 assert (rm_opnd.type == OPND_MEM);
673 uint8_t scale = rm_opnd.as.mem.
scale_exp;
686 uint8_t sib_byte = (scale << 6) + (index << 3) + (base);
687 cb_write_byte(cb, sib_byte);
691 if (rm_opnd.type == OPND_MEM)
693 uint32_t dsize = disp_size(rm_opnd);
695 cb_write_int(cb, rm_opnd.as.mem.
disp, dsize);
700static void write_rm_unary(
704 uint8_t opMemRegPref,
713 if (opnd.type == OPND_REG || opnd.type == OPND_MEM)
714 opndSize = opnd.num_bits;
716 rb_bug(
"yjit: invalid operand");
718 assert (opndSize == 8 || opndSize == 16 || opndSize == 32 || opndSize == 64);
719 bool szPref = opndSize == 16;
720 bool rexW = opndSize == 64;
723 cb_write_rm(cb,
false,
false, NO_OPND, opnd, opExt, 1, opMemReg8);
725 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd, opExt, 1, opMemRegPref);
729static void cb_write_rm_multi(
733 uint8_t opMemRegPref,
735 uint8_t opRegMemPref,
743 assert (opnd0.type == OPND_REG || opnd0.type == OPND_MEM);
754 uint32_t opndSize = opnd0.num_bits;
757 if (opnd1.type == OPND_REG || opnd1.type == OPND_MEM)
759 assert (opnd1.num_bits == opndSize &&
"operand size mismatch");
761 else if (opnd1.type == OPND_IMM)
763 assert (opnd1.num_bits <= opndSize);
766 assert (opndSize == 8 || opndSize == 16 || opndSize == 32 || opndSize == 64);
767 bool szPref = opndSize == 16;
768 bool rexW = opndSize == 64;
771 if ((opnd0.type == OPND_MEM && opnd1.type == OPND_REG) ||
772 (opnd0.type == OPND_REG && opnd1.type == OPND_REG))
776 cb_write_rm(cb,
false,
false, opnd1, opnd0, 0xFF, 1, opMemReg8);
778 cb_write_rm(cb, szPref, rexW, opnd1, opnd0, 0xFF, 1, opMemRegPref);
782 else if (opnd0.type == OPND_REG && opnd1.type == OPND_MEM)
786 cb_write_rm(cb,
false,
false, opnd0, opnd1, 0xFF, 1, opRegMem8);
788 cb_write_rm(cb, szPref, rexW, opnd0, opnd1, 0xFF, 1, opRegMemPref);
792 else if (opnd1.type == OPND_IMM)
795 if (opnd1.num_bits <= 8)
798 cb_write_rm(cb,
false,
false, NO_OPND, opnd0, opExtImm, 1, opMemImm8);
800 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExtImm, 1, opMemImmSml);
802 cb_write_int(cb, opnd1.as.imm, 8);
806 else if (opnd1.num_bits <= 32)
808 assert (opnd1.num_bits <= opndSize &&
"immediate too large for dst");
809 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExtImm, 1, opMemImmLrg);
810 cb_write_int(cb, opnd1.as.imm, (opndSize > 32)? 32:opndSize);
816 assert (
false &&
"immediate value too large");
823 assert (
false &&
"invalid operand combination");
828static void cb_write_shift(
831 uint8_t opMemOnePref,
833 uint8_t opMemImmPref,
843 if (opnd0.type == OPND_REG || opnd0.type == OPND_MEM)
844 opndSize = opnd0.num_bits;
846 rb_bug(
"yjit: shift: invalid first operand");
848 assert (opndSize == 16 || opndSize == 32 || opndSize == 64);
849 bool szPref = opndSize == 16;
850 bool rexW = opndSize == 64;
852 if (opnd1.type == OPND_IMM)
854 if (opnd1.as.imm == 1)
856 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExt, 1, opMemOnePref);
860 assert (opnd1.num_bits <= 8);
861 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExt, 1, opMemImmPref);
862 cb_write_byte(cb, (uint8_t)opnd1.as.imm);
879static void cb_write_jcc(
codeblock_t *cb,
const char *mnem, uint8_t op0, uint8_t op1, uint32_t label_idx)
885 cb_write_byte(cb, op0);
886 cb_write_byte(cb, op1);
889 cb_label_ref(cb, label_idx);
892 cb_write_int(cb, 0, 32);
896static void cb_write_jcc_ptr(
codeblock_t *cb,
const char *mnem, uint8_t op0, uint8_t op1, uint8_t *dst_ptr)
902 cb_write_byte(cb, op0);
903 cb_write_byte(cb, op1);
906 uint8_t *end_ptr = cb_get_ptr(cb, cb->write_pos + 4);
909 int64_t rel64 = (int64_t)(dst_ptr - end_ptr);
910 if (rel64 >= INT32_MIN && rel64 <= INT32_MAX) {
912 cb_write_int(cb, (int32_t)rel64, 32);
916 cb->dropped_bytes =
true;
925 assert (dst.type == OPND_REG);
926 assert (src.type == OPND_REG || src.type == OPND_MEM);
927 assert (dst.num_bits >= 16 &&
"invalid dst reg size in cmov");
929 bool szPref = dst.num_bits == 16;
930 bool rexW = dst.num_bits == 64;
932 cb_write_rm(cb, szPref, rexW, dst, src, 0xFF, 2, 0x0F, opcode1);
974static void call_rel32(
codeblock_t *cb, int32_t rel32)
979 cb_write_byte(cb, 0xE8);
982 cb_write_int(cb, (int32_t)rel32, 32);
988 assert (scratch_reg.type == OPND_REG);
991 uint8_t *end_ptr = cb_get_ptr(cb, cb->write_pos + 5);
994 int64_t rel64 = (int64_t)(dst_ptr - end_ptr);
997 if (rel64 >= INT32_MIN && rel64 <= INT32_MAX) {
998 call_rel32(cb, (int32_t)rel64);
1003 mov(cb, scratch_reg, const_ptr_opnd(dst_ptr));
1004 call(cb, scratch_reg);
1008void call_label(
codeblock_t *cb, uint32_t label_idx)
1013 cb_write_byte(cb, 0xE8);
1016 cb_label_ref(cb, label_idx);
1019 cb_write_int(cb, 0, 32);
1026 cb_write_rm(cb,
false,
false, NO_OPND, opnd, 2, 1, 0xFF);
1084 cb_write_byte(cb, 0x99);
1091 cb_write_bytes(cb, 2, 0x48, 0x99);
1098 cb_write_byte(cb, 0xCC);
1200void ja_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"ja" , 0x0F, 0x87, label_idx); }
1201void jae_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jae" , 0x0F, 0x83, label_idx); }
1202void jb_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jb" , 0x0F, 0x82, label_idx); }
1203void jbe_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jbe" , 0x0F, 0x86, label_idx); }
1204void jc_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jc" , 0x0F, 0x82, label_idx); }
1205void je_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"je" , 0x0F, 0x84, label_idx); }
1206void jg_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jg" , 0x0F, 0x8F, label_idx); }
1207void jge_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jge" , 0x0F, 0x8D, label_idx); }
1208void jl_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jl" , 0x0F, 0x8C, label_idx); }
1209void jle_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jle" , 0x0F, 0x8E, label_idx); }
1210void jna_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jna" , 0x0F, 0x86, label_idx); }
1211void jnae_label(
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnae", 0x0F, 0x82, label_idx); }
1212void jnb_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnb" , 0x0F, 0x83, label_idx); }
1213void jnbe_label(
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnbe", 0x0F, 0x87, label_idx); }
1214void jnc_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnc" , 0x0F, 0x83, label_idx); }
1215void jne_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jne" , 0x0F, 0x85, label_idx); }
1216void jng_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jng" , 0x0F, 0x8E, label_idx); }
1217void jnge_label(
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnge", 0x0F, 0x8C, label_idx); }
1218void jnl_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnl" , 0x0F, 0x8D, label_idx); }
1219void jnle_label(
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnle", 0x0F, 0x8F, label_idx); }
1220void jno_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jno" , 0x0F, 0x81, label_idx); }
1221void jnp_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnp" , 0x0F, 0x8b, label_idx); }
1222void jns_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jns" , 0x0F, 0x89, label_idx); }
1223void jnz_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jnz" , 0x0F, 0x85, label_idx); }
1224void jo_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jo" , 0x0F, 0x80, label_idx); }
1225void jp_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jp" , 0x0F, 0x8A, label_idx); }
1226void jpe_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jpe" , 0x0F, 0x8A, label_idx); }
1227void jpo_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jpo" , 0x0F, 0x8B, label_idx); }
1228void js_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"js" , 0x0F, 0x88, label_idx); }
1229void jz_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jz" , 0x0F, 0x84, label_idx); }
1230void jmp_label (
codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb,
"jmp" , 0xFF, 0xE9, label_idx); }
1233void ja_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"ja" , 0x0F, 0x87, ptr); }
1234void jae_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jae" , 0x0F, 0x83, ptr); }
1235void jb_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jb" , 0x0F, 0x82, ptr); }
1236void jbe_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jbe" , 0x0F, 0x86, ptr); }
1237void jc_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jc" , 0x0F, 0x82, ptr); }
1238void je_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"je" , 0x0F, 0x84, ptr); }
1239void jg_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jg" , 0x0F, 0x8F, ptr); }
1240void jge_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jge" , 0x0F, 0x8D, ptr); }
1241void jl_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jl" , 0x0F, 0x8C, ptr); }
1242void jle_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jle" , 0x0F, 0x8E, ptr); }
1243void jna_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jna" , 0x0F, 0x86, ptr); }
1244void jnae_ptr(
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnae", 0x0F, 0x82, ptr); }
1245void jnb_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnb" , 0x0F, 0x83, ptr); }
1246void jnbe_ptr(
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnbe", 0x0F, 0x87, ptr); }
1247void jnc_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnc" , 0x0F, 0x83, ptr); }
1248void jne_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jne" , 0x0F, 0x85, ptr); }
1249void jng_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jng" , 0x0F, 0x8E, ptr); }
1250void jnge_ptr(
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnge", 0x0F, 0x8C, ptr); }
1251void jnl_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnl" , 0x0F, 0x8D, ptr); }
1252void jnle_ptr(
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnle", 0x0F, 0x8F, ptr); }
1253void jno_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jno" , 0x0F, 0x81, ptr); }
1254void jnp_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnp" , 0x0F, 0x8b, ptr); }
1255void jns_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jns" , 0x0F, 0x89, ptr); }
1256void jnz_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jnz" , 0x0F, 0x85, ptr); }
1257void jo_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jo" , 0x0F, 0x80, ptr); }
1258void jp_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jp" , 0x0F, 0x8A, ptr); }
1259void jpe_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jpe" , 0x0F, 0x8A, ptr); }
1260void jpo_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jpo" , 0x0F, 0x8B, ptr); }
1261void js_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"js" , 0x0F, 0x88, ptr); }
1262void jz_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jz" , 0x0F, 0x84, ptr); }
1263void jmp_ptr (
codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb,
"jmp" , 0xFF, 0xE9, ptr); }
1269 cb_write_rm(cb,
false,
false, NO_OPND, opnd, 4, 1, 0xFF);
1276 cb_write_byte(cb, 0xE9);
1277 cb_write_int(cb, offset, 32);
1284 assert (dst.num_bits == 64);
1285 cb_write_rm(cb,
false,
true, dst, src, 0xFF, 1, 0x8D);
1292zero_extendable_32bit(uint64_t number)
1294 return number <= UINT32_MAX && (number & (1ull << 31ull)) == 0;
1301 if (src.type == OPND_IMM)
1306 if (dst.type == OPND_REG)
1309 src.num_bits <= dst.num_bits ||
1310 unsig_imm_size(src.as.imm) <= dst.num_bits
1317 if (dst.num_bits == 64 && zero_extendable_32bit(src.as.unsig_imm)) {
1318 if (rex_needed(dst))
1319 cb_write_rex(cb,
false, 0, 0, dst.as.reg.reg_no);
1320 cb_write_opcode(cb, 0xB8, dst);
1321 cb_write_int(cb, src.as.imm, 32);
1324 if (dst.num_bits == 16)
1325 cb_write_byte(cb, 0x66);
1327 if (rex_needed(dst) || dst.num_bits == 64)
1328 cb_write_rex(cb, dst.num_bits == 64, 0, 0, dst.as.reg.reg_no);
1330 cb_write_opcode(cb, (dst.num_bits == 8)? 0xB0:0xB8, dst);
1332 cb_write_int(cb, src.as.imm, dst.num_bits);
1337 else if (dst.type == OPND_MEM)
1339 assert (src.num_bits <= dst.num_bits);
1341 if (dst.num_bits == 8)
1342 cb_write_rm(cb,
false,
false, NO_OPND, dst, 0xFF, 1, 0xC6);
1344 cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, NO_OPND, dst, 0, 1, 0xC7);
1346 const uint32_t output_num_bits = (dst.num_bits > 32u) ? 32u : dst.num_bits;
1348 assert (sig_imm_size(src.as.imm) <= output_num_bits);
1349 cb_write_int(cb, src.as.imm, output_num_bits);
1379 assert (dst.type == OPND_REG);
1380 assert (src.type == OPND_REG || src.type == OPND_MEM);
1381 assert (src.num_bits < dst.num_bits);
1385 if (src.num_bits == 8)
1387 cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, dst, src, 0xFF, 2, 0x0F, 0xBE);
1389 else if (src.num_bits == 16)
1391 cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, dst, src, 0xFF, 2, 0x0F, 0xBF);
1393 else if (src.num_bits == 32)
1395 cb_write_rm(cb,
false,
true, dst, src, 0xFF, 1, 0x63);
1465 cb_write_byte(cb, 0x90);
1470 cb_write_bytes(cb, 2, 0x66,0x90);
1475 cb_write_bytes(cb, 3, 0x0F,0x1F,0x00);
1480 cb_write_bytes(cb, 4, 0x0F,0x1F,0x40,0x00);
1485 cb_write_bytes(cb, 5, 0x0F,0x1F,0x44,0x00,0x00);
1490 cb_write_bytes(cb, 6, 0x66,0x0F,0x1F,0x44,0x00,0x00);
1495 cb_write_bytes(cb, 7, 0x0F,0x1F,0x80,0x00,0x00,0x00,0x00);
1500 cb_write_bytes(cb, 8, 0x0F,0x1F,0x84,0x00,0x00,0x00,0x00,0x00);
1505 cb_write_bytes(cb, 9, 0x66,0x0F,0x1F,0x84,0x00,0x00,0x00,0x00,0x00);
1510 uint32_t written = 0;
1511 while (written + 9 <= length)
1516 nop(cb, length - written);
1557 assert (opnd.num_bits == 64);
1561 if (opnd.type == OPND_REG) {
1562 if (rex_needed(opnd))
1563 cb_write_rex(cb,
false, 0, 0, opnd.as.reg.reg_no);
1564 cb_write_opcode(cb, 0x58, opnd);
1566 else if (opnd.type == OPND_MEM) {
1567 cb_write_rm(cb,
false,
false, NO_OPND, opnd, 0, 1, 0x8F);
1570 assert(
false &&
"unexpected operand type");
1580 cb_write_bytes(cb, 2, 0x48, 0x9D);
1586 assert (opnd.num_bits == 64);
1590 if (opnd.type == OPND_REG) {
1591 if (rex_needed(opnd))
1592 cb_write_rex(cb,
false, 0, 0, opnd.as.reg.reg_no);
1593 cb_write_opcode(cb, 0x50, opnd);
1595 else if (opnd.type == OPND_MEM) {
1596 cb_write_rm(cb,
false,
false, NO_OPND, opnd, 6, 1, 0xFF);
1599 assert(
false &&
"unexpected operand type");
1607 cb_write_byte(cb, 0x9C);
1614 cb_write_byte(cb, 0xC3);
1698 assert (rm_opnd.type == OPND_REG || rm_opnd.type == OPND_MEM);
1699 assert (test_opnd.type == OPND_REG || test_opnd.type == OPND_IMM);
1702 if (test_opnd.type == OPND_IMM)
1706 if (imm_opnd.as.imm >= 0)
1708 assert (unsig_imm_size(imm_opnd.as.unsig_imm) <= 32);
1709 assert (unsig_imm_size(imm_opnd.as.unsig_imm) <= rm_opnd.num_bits);
1712 rm_opnd = resize_opnd(rm_opnd, unsig_imm_size(imm_opnd.as.unsig_imm));
1714 if (rm_opnd.num_bits == 8)
1716 cb_write_rm(cb,
false,
false, NO_OPND, rm_opnd, 0x00, 1, 0xF6);
1717 cb_write_int(cb, imm_opnd.as.imm, rm_opnd.num_bits);
1721 cb_write_rm(cb, rm_opnd.num_bits == 16,
false, NO_OPND, rm_opnd, 0x00, 1, 0xF7);
1722 cb_write_int(cb, imm_opnd.as.imm, rm_opnd.num_bits);
1728 assert (imm_opnd.as.imm < 0);
1729 assert (sig_imm_size(imm_opnd.as.imm) <= 32);
1730 assert (rm_opnd.num_bits == 64);
1731 cb_write_rm(cb,
false,
true, NO_OPND, rm_opnd, 0x00, 1, 0xF7);
1732 cb_write_int(cb, imm_opnd.as.imm, 32);
1737 assert (test_opnd.num_bits == rm_opnd.num_bits);
1739 if (rm_opnd.num_bits == 8)
1741 cb_write_rm(cb,
false,
false, test_opnd, rm_opnd, 0xFF, 1, 0x84);
1745 cb_write_rm(cb, rm_opnd.num_bits == 16, rm_opnd.num_bits == 64, test_opnd, rm_opnd, 0xFF, 1, 0x85);
1753 cb_write_bytes(cb, 2, 0x0F, 0x0B);
1759 assert (rm_opnd.num_bits == 64);
1760 assert (r_opnd.num_bits == 64);
1761 assert (rm_opnd.type == OPND_REG);
1762 assert (r_opnd.type == OPND_REG);
1765 if (rm_opnd.type == OPND_REG && rm_opnd.as.reg.reg_no == RAX.as.reg.reg_no)
1768 cb_write_rex(cb, rm_opnd.num_bits == 64, 0, 0, r_opnd.as.reg.reg_no);
1771 cb_write_byte(cb, 0x90 + (r_opnd.as.reg.reg_no & 7));
1775 cb_write_rm(cb, rm_opnd.num_bits == 16, rm_opnd.num_bits == 64, r_opnd, rm_opnd, 0xFF, 1, 0x87);
1801 cb_write_byte(cb, 0xF0);
1806 if (mprotect(cb->mem_block_, cb->mem_size, PROT_READ | PROT_WRITE)) {
1807 fprintf(stderr,
"Couldn't make JIT page (%p) writeable, errno: %s", (
void *)cb->mem_block_, strerror(errno));
1812void cb_mark_position_writeable(
codeblock_t * cb, uint32_t write_pos)
1814 uint32_t pagesize = (uint32_t)sysconf(_SC_PAGESIZE);
1815 uint32_t aligned_position = (write_pos / pagesize) * pagesize;
1817 if (cb->current_aligned_write_pos != aligned_position) {
1818 cb->current_aligned_write_pos = aligned_position;
1819 void *
const page_addr = cb_get_ptr(cb, aligned_position);
1820 if (mprotect(page_addr, pagesize, PROT_READ | PROT_WRITE)) {
1821 fprintf(stderr,
"Couldn't make JIT page (%p) writeable, errno: %s", page_addr, strerror(errno));
1829 cb->current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
1830 if (mprotect(cb->mem_block_, cb->mem_size, PROT_READ | PROT_EXEC)) {
1831 fprintf(stderr,
"Couldn't make JIT page (%p) executable, errno: %s", (
void *)cb->mem_block_, strerror(errno));
void rb_bug(const char *fmt,...)
Interpreter panic switch.
uint8_t base_reg_no
Base register number.
bool has_idx
Has index register flag.
bool is_iprel
IP-relative addressing flag.
uint8_t scale_exp
SIB scale exponent value (power of two, two bits)
int32_t disp
Constant displacement from the base, not scaled.
uint8_t idx_reg_no
Index register number.