Ruby 3.1.3p185 (2022-11-24 revision 1a6b16756e0ba6b95ab71a441357ed5484e33498)
yjit_asm.c
1// This file is a fragment of the yjit.o compilation unit. See yjit.c.
2//
3// Note that the definition for some of these functions don't specify
4// static inline, but their declaration in yjit_asm.h do. The resulting
5// linkage is the same as if they both specify. The relevant sections in
6// N1256 is 6.2.2p4, 6.2.2p5, and 6.7.4p5.
7#include <stdio.h>
8#include <stdlib.h>
9#include <string.h>
10#include <stdarg.h>
11#include <stdint.h>
12#include <assert.h>
13#include <errno.h>
14
15// For mmapp(), sysconf()
16#ifndef _WIN32
17#include <unistd.h>
18#include <sys/mman.h>
19#endif
20
21#include "yjit_asm.h"
22
23// Compute the number of bits needed to encode a signed value
24uint32_t sig_imm_size(int64_t imm)
25{
26 // Compute the smallest size this immediate fits in
27 if (imm >= INT8_MIN && imm <= INT8_MAX)
28 return 8;
29 if (imm >= INT16_MIN && imm <= INT16_MAX)
30 return 16;
31 if (imm >= INT32_MIN && imm <= INT32_MAX)
32 return 32;
33
34 return 64;
35}
36
37// Compute the number of bits needed to encode an unsigned value
38uint32_t unsig_imm_size(uint64_t imm)
39{
40 // Compute the smallest size this immediate fits in
41 if (imm <= UINT8_MAX)
42 return 8;
43 else if (imm <= UINT16_MAX)
44 return 16;
45 else if (imm <= UINT32_MAX)
46 return 32;
47
48 return 64;
49}
50
51x86opnd_t mem_opnd(uint32_t num_bits, x86opnd_t base_reg, int32_t disp)
52{
53 bool is_iprel = base_reg.as.reg.reg_type == REG_IP;
54
55 x86opnd_t opnd = {
56 OPND_MEM,
57 num_bits,
58 .as.mem = { base_reg.as.reg.reg_no, 0, 0, false, is_iprel, disp }
59 };
60
61 return opnd;
62}
63
64x86opnd_t mem_opnd_sib(uint32_t num_bits, x86opnd_t base_reg, x86opnd_t index_reg, int32_t scale, int32_t disp)
65{
66 uint8_t scale_exp;
67 switch (scale) {
68 case 8:
69 scale_exp = 3;
70 break;
71 case 4:
72 scale_exp = 2;
73 break;
74 case 2:
75 scale_exp = 1;
76 break;
77 case 1:
78 scale_exp = 0;
79 break;
80 default:
81 rb_bug("yjit: scale not one of 1,2,4,8");
82 break;
83 }
84
85 bool is_iprel = base_reg.as.reg.reg_type == REG_IP;
86
87 x86opnd_t opnd = {
88 OPND_MEM,
89 num_bits,
90 .as.mem = {
91 .base_reg_no = base_reg.as.reg.reg_no,
92 .idx_reg_no = index_reg.as.reg.reg_no,
93 .has_idx = 1,
94 .scale_exp = scale_exp,
95 .is_iprel = is_iprel,
96 .disp = disp
97 }
98 };
99
100 return opnd;
101}
102
103static x86opnd_t resize_opnd(x86opnd_t opnd, uint32_t num_bits)
104{
105 assert (num_bits % 8 == 0);
106 x86opnd_t sub = opnd;
107 sub.num_bits = num_bits;
108 return sub;
109}
110
111x86opnd_t imm_opnd(int64_t imm)
112{
113 x86opnd_t opnd = {
114 OPND_IMM,
115 sig_imm_size(imm),
116 .as.imm = imm
117 };
118
119 return opnd;
120}
121
122x86opnd_t const_ptr_opnd(const void *ptr)
123{
124 x86opnd_t opnd = {
125 OPND_IMM,
126 64,
127 .as.unsig_imm = (uint64_t)ptr
128 };
129
130 return opnd;
131}
132
133// Align the current write position to a multiple of bytes
134static uint8_t *align_ptr(uint8_t *ptr, uint32_t multiple)
135{
136 // Compute the pointer modulo the given alignment boundary
137 uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
138
139 // If the pointer is already aligned, stop
140 if (rem == 0)
141 return ptr;
142
143 // Pad the pointer by the necessary amount to align it
144 uint32_t pad = multiple - rem;
145
146 return ptr + pad;
147}
148
149// Allocate a block of executable memory
150static uint8_t *alloc_exec_mem(uint32_t mem_size)
151{
152#ifndef _WIN32
153 uint8_t *mem_block;
154
155 // On Linux
156 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
157 // Align the requested address to page size
158 uint32_t page_size = (uint32_t)sysconf(_SC_PAGESIZE);
159 uint8_t *req_addr = align_ptr((uint8_t*)&alloc_exec_mem, page_size);
160
161 do {
162 // Try to map a chunk of memory as executable
163 mem_block = (uint8_t*)mmap(
164 (void*)req_addr,
165 mem_size,
166 PROT_READ | PROT_EXEC,
167 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
168 -1,
169 0
170 );
171
172 // If we succeeded, stop
173 if (mem_block != MAP_FAILED) {
174 break;
175 }
176
177 // +4MB
178 req_addr += 4 * 1024 * 1024;
179 } while (req_addr < (uint8_t*)&alloc_exec_mem + INT32_MAX);
180
181 // On MacOS and other platforms
182 #else
183 // Try to map a chunk of memory as executable
184 mem_block = (uint8_t*)mmap(
185 (void*)alloc_exec_mem,
186 mem_size,
187 PROT_READ | PROT_EXEC,
188 MAP_PRIVATE | MAP_ANONYMOUS,
189 -1,
190 0
191 );
192 #endif
193
194 // Fallback
195 if (mem_block == MAP_FAILED) {
196 // Try again without the address hint (e.g., valgrind)
197 mem_block = (uint8_t*)mmap(
198 NULL,
199 mem_size,
200 PROT_READ | PROT_EXEC,
201 MAP_PRIVATE | MAP_ANONYMOUS,
202 -1,
203 0
204 );
205 }
206
207 // Check that the memory mapping was successful
208 if (mem_block == MAP_FAILED) {
209 perror("mmap call failed");
210 exit(-1);
211 }
212
213 codeblock_t block;
214 codeblock_t *cb = &block;
215
216 cb_init(cb, mem_block, mem_size);
217
218 // Fill the executable memory with PUSH DS (0x1E) so that
219 // executing uninitialized memory will fault with #UD in
220 // 64-bit mode.
221 cb_mark_all_writeable(cb);
222 memset(mem_block, 0x1E, mem_size);
223 cb_mark_all_executable(cb);
224
225 return mem_block;
226#else
227 // Windows not supported for now
228 return NULL;
229#endif
230}
231
232// Initialize a code block object
233void cb_init(codeblock_t *cb, uint8_t *mem_block, uint32_t mem_size)
234{
235 assert (mem_block);
236 cb->mem_block_ = mem_block;
237 cb->mem_size = mem_size;
238 cb->write_pos = 0;
239 cb->num_labels = 0;
240 cb->num_refs = 0;
241 cb->current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
242}
243
244// Set the current write position
245void cb_set_pos(codeblock_t *cb, uint32_t pos)
246{
247 // Assert here since while assembler functions do bounds checking, there is
248 // nothing stopping users from taking out an out-of-bounds pointer and
249 // doing bad accesses with it.
250 assert (pos < cb->mem_size);
251 cb->write_pos = pos;
252}
253
254// Align the current write position to a multiple of bytes
255void cb_align_pos(codeblock_t *cb, uint32_t multiple)
256{
257 // Compute the pointer modulo the given alignment boundary
258 uint8_t *ptr = cb_get_write_ptr(cb);
259 uint8_t *aligned_ptr = align_ptr(ptr, multiple);
260 const uint32_t write_pos = cb->write_pos;
261
262 // Pad the pointer by the necessary amount to align it
263 ptrdiff_t pad = aligned_ptr - ptr;
264 cb_set_pos(cb, write_pos + (int32_t)pad);
265}
266
267// Set the current write position from a pointer
268void cb_set_write_ptr(codeblock_t *cb, uint8_t *code_ptr)
269{
270 intptr_t pos = code_ptr - cb->mem_block_;
271 assert (pos < cb->mem_size);
272 cb_set_pos(cb, (uint32_t)pos);
273}
274
275// Get a direct pointer into the executable memory block
276uint8_t *cb_get_ptr(const codeblock_t *cb, uint32_t index)
277{
278 if (index < cb->mem_size) {
279 return &cb->mem_block_[index];
280 }
281 else {
282 return NULL;
283 }
284}
285
286// Get a direct pointer to the current write position
287uint8_t *cb_get_write_ptr(const codeblock_t *cb)
288{
289 return cb_get_ptr(cb, cb->write_pos);
290}
291
292// Write a byte at the current position
293void cb_write_byte(codeblock_t *cb, uint8_t byte)
294{
295 assert (cb->mem_block_);
296 if (cb->write_pos < cb->mem_size) {
297 cb_mark_position_writeable(cb, cb->write_pos);
298 cb->mem_block_[cb->write_pos] = byte;
299 cb->write_pos++;
300 }
301 else {
302 cb->dropped_bytes = true;
303 }
304}
305
306// Write multiple bytes starting from the current position
307void cb_write_bytes(codeblock_t *cb, uint32_t num_bytes, ...)
308{
309 va_list va;
310 va_start(va, num_bytes);
311
312 for (uint32_t i = 0; i < num_bytes; ++i)
313 {
314 uint8_t byte = va_arg(va, int);
315 cb_write_byte(cb, byte);
316 }
317
318 va_end(va);
319}
320
321// Write a signed integer over a given number of bits at the current position
322void cb_write_int(codeblock_t *cb, uint64_t val, uint32_t num_bits)
323{
324 assert (num_bits > 0);
325 assert (num_bits % 8 == 0);
326
327 // Switch on the number of bits
328 switch (num_bits) {
329 case 8:
330 cb_write_byte(cb, (uint8_t)val);
331 break;
332
333 case 16:
334 cb_write_bytes(
335 cb,
336 2,
337 (uint8_t)((val >> 0) & 0xFF),
338 (uint8_t)((val >> 8) & 0xFF)
339 );
340 break;
341
342 case 32:
343 cb_write_bytes(
344 cb,
345 4,
346 (uint8_t)((val >> 0) & 0xFF),
347 (uint8_t)((val >> 8) & 0xFF),
348 (uint8_t)((val >> 16) & 0xFF),
349 (uint8_t)((val >> 24) & 0xFF)
350 );
351 break;
352
353 default:
354 {
355 // Compute the size in bytes
356 uint32_t num_bytes = num_bits / 8;
357
358 // Write out the bytes
359 for (uint32_t i = 0; i < num_bytes; ++i)
360 {
361 uint8_t byte_val = (uint8_t)(val & 0xFF);
362 cb_write_byte(cb, byte_val);
363 val >>= 8;
364 }
365 }
366 }
367}
368
369// Allocate a new label with a given name
370uint32_t cb_new_label(codeblock_t *cb, const char *name)
371{
372 //if (hasASM)
373 // writeString(to!string(label) ~ ":");
374
375 assert (cb->num_labels < MAX_LABELS);
376
377 // Allocate the new label
378 uint32_t label_idx = cb->num_labels++;
379
380 // This label doesn't have an address yet
381 cb->label_addrs[label_idx] = 0;
382 cb->label_names[label_idx] = name;
383
384 return label_idx;
385}
386
387// Write a label at the current address
388void cb_write_label(codeblock_t *cb, uint32_t label_idx)
389{
390 assert (label_idx < MAX_LABELS);
391 cb->label_addrs[label_idx] = cb->write_pos;
392}
393
394// Add a label reference at the current write position
395void cb_label_ref(codeblock_t *cb, uint32_t label_idx)
396{
397 assert (label_idx < MAX_LABELS);
398 assert (cb->num_refs < MAX_LABEL_REFS);
399
400 // Keep track of the reference
401 cb->label_refs[cb->num_refs] = (labelref_t){ cb->write_pos, label_idx };
402 cb->num_refs++;
403}
404
405// Link internal label references
406void cb_link_labels(codeblock_t *cb)
407{
408 uint32_t orig_pos = cb->write_pos;
409
410 // For each label reference
411 for (uint32_t i = 0; i < cb->num_refs; ++i)
412 {
413 uint32_t ref_pos = cb->label_refs[i].pos;
414 uint32_t label_idx = cb->label_refs[i].label_idx;
415 assert (ref_pos < cb->mem_size);
416 assert (label_idx < MAX_LABELS);
417
418 uint32_t label_addr = cb->label_addrs[label_idx];
419 assert (label_addr < cb->mem_size);
420
421 // Compute the offset from the reference's end to the label
422 int64_t offset = (int64_t)label_addr - (int64_t)(ref_pos + 4);
423
424 cb_set_pos(cb, ref_pos);
425 cb_write_int(cb, offset, 32);
426 }
427
428 cb->write_pos = orig_pos;
429
430 // Clear the label positions and references
431 cb->num_labels = 0;
432 cb->num_refs = 0;
433}
434
435// Check if an operand needs a REX byte to be encoded
436static bool rex_needed(x86opnd_t opnd)
437{
438 if (opnd.type == OPND_NONE || opnd.type == OPND_IMM)
439 {
440 return false;
441 }
442
443 if (opnd.type == OPND_REG)
444 {
445 return (
446 opnd.as.reg.reg_no > 7 ||
447 (opnd.num_bits == 8 && opnd.as.reg.reg_no >= 4 && opnd.as.reg.reg_no <= 7)
448 );
449 }
450
451 if (opnd.type == OPND_MEM)
452 {
453 return (opnd.as.mem.base_reg_no > 7) || (opnd.as.mem.has_idx && opnd.as.mem.idx_reg_no > 7);
454 }
455
456 rb_bug("unreachable");
457}
458
459// Check if an SIB byte is needed to encode this operand
460static bool sib_needed(x86opnd_t opnd)
461{
462 if (opnd.type != OPND_MEM)
463 return false;
464
465 return (
466 opnd.as.mem.has_idx ||
467 opnd.as.mem.base_reg_no == RSP.as.reg.reg_no ||
468 opnd.as.mem.base_reg_no == R12.as.reg.reg_no
469 );
470}
471
472// Compute the size of the displacement field needed for a memory operand
473static uint32_t disp_size(x86opnd_t opnd)
474{
475 assert (opnd.type == OPND_MEM);
476
477 // If using RIP as the base, use disp32
478 if (opnd.as.mem.is_iprel)
479 {
480 return 32;
481 }
482
483 // Compute the required displacement size
484 if (opnd.as.mem.disp != 0)
485 {
486 uint32_t num_bits = sig_imm_size(opnd.as.mem.disp);
487 assert (num_bits <= 32 && "displacement does not fit in 32 bits");
488
489 // x86 can only encode 8-bit and 32-bit displacements
490 if (num_bits == 16)
491 num_bits = 32;;
492
493 return num_bits;
494 }
495
496 // If EBP or RBP or R13 is used as the base, displacement must be encoded
497 if (opnd.as.mem.base_reg_no == RBP.as.reg.reg_no ||
498 opnd.as.mem.base_reg_no == R13.as.reg.reg_no)
499 {
500 return 8;
501 }
502
503 return 0;
504}
505
506// Write the REX byte
507static void cb_write_rex(
508 codeblock_t *cb,
509 bool w_flag,
510 uint8_t reg_no,
511 uint8_t idx_reg_no,
512 uint8_t rm_reg_no
513)
514{
515 // 0 1 0 0 w r x b
516 // w - 64-bit operand size flag
517 // r - MODRM.reg extension
518 // x - SIB.index extension
519 // b - MODRM.rm or SIB.base extension
520 uint8_t w = w_flag? 1:0;
521 uint8_t r = (reg_no & 8)? 1:0;
522 uint8_t x = (idx_reg_no & 8)? 1:0;
523 uint8_t b = (rm_reg_no & 8)? 1:0;
524
525 // Encode and write the REX byte
526 uint8_t rexByte = 0x40 + (w << 3) + (r << 2) + (x << 1) + (b);
527 cb_write_byte(cb, rexByte);
528}
529
530// Write an opcode byte with an embedded register operand
531static void cb_write_opcode(codeblock_t *cb, uint8_t opcode, x86opnd_t reg)
532{
533 // Write the reg field into the opcode byte
534 uint8_t op_byte = opcode | (reg.as.reg.reg_no & 7);
535 cb_write_byte(cb, op_byte);
536}
537
538// Encode an RM instruction
539static void cb_write_rm(
540 codeblock_t *cb,
541 bool szPref,
542 bool rexW,
543 x86opnd_t r_opnd,
544 x86opnd_t rm_opnd,
545 uint8_t opExt,
546 uint32_t op_len,
547 ...)
548{
549 assert (op_len > 0 && op_len <= 3);
550 assert (r_opnd.type == OPND_REG || r_opnd.type == OPND_NONE);
551
552 // Flag to indicate the REX prefix is needed
553 bool need_rex = rexW || rex_needed(r_opnd) || rex_needed(rm_opnd);
554
555 // Flag to indicate SIB byte is needed
556 bool need_sib = sib_needed(r_opnd) || sib_needed(rm_opnd);
557
558 // Add the operand-size prefix, if needed
559 if (szPref == true)
560 cb_write_byte(cb, 0x66);
561
562 // Add the REX prefix, if needed
563 if (need_rex)
564 {
565 // 0 1 0 0 w r x b
566 // w - 64-bit operand size flag
567 // r - MODRM.reg extension
568 // x - SIB.index extension
569 // b - MODRM.rm or SIB.base extension
570
571 uint8_t w = rexW? 1:0;
572
573 uint8_t r;
574 if (r_opnd.type != OPND_NONE)
575 r = (r_opnd.as.reg.reg_no & 8)? 1:0;
576 else
577 r = 0;
578
579 uint8_t x;
580 if (need_sib && rm_opnd.as.mem.has_idx)
581 x = (rm_opnd.as.mem.idx_reg_no & 8)? 1:0;
582 else
583 x = 0;
584
585 uint8_t b;
586 if (rm_opnd.type == OPND_REG)
587 b = (rm_opnd.as.reg.reg_no & 8)? 1:0;
588 else if (rm_opnd.type == OPND_MEM)
589 b = (rm_opnd.as.mem.base_reg_no & 8)? 1:0;
590 else
591 b = 0;
592
593 // Encode and write the REX byte
594 uint8_t rex_byte = 0x40 + (w << 3) + (r << 2) + (x << 1) + (b);
595 cb_write_byte(cb, rex_byte);
596 }
597
598 // Write the opcode bytes to the code block
599 va_list va;
600 va_start(va, op_len);
601 for (uint32_t i = 0; i < op_len; ++i)
602 {
603 uint8_t byte = va_arg(va, int);
604 cb_write_byte(cb, byte);
605 }
606 va_end(va);
607
608 // MODRM.mod (2 bits)
609 // MODRM.reg (3 bits)
610 // MODRM.rm (3 bits)
611
612 assert (
613 !(opExt != 0xFF && r_opnd.type != OPND_NONE) &&
614 "opcode extension and register operand present"
615 );
616
617 // Encode the mod field
618 uint8_t mod;
619 if (rm_opnd.type == OPND_REG)
620 {
621 mod = 3;
622 }
623 else
624 {
625 uint32_t dsize = disp_size(rm_opnd);
626 if (dsize == 0 || rm_opnd.as.mem.is_iprel)
627 mod = 0;
628 else if (dsize == 8)
629 mod = 1;
630 else if (dsize == 32)
631 mod = 2;
632 else
633 rb_bug("unreachable");
634 }
635
636 // Encode the reg field
637 uint8_t reg;
638 if (opExt != 0xFF)
639 reg = opExt;
640 else if (r_opnd.type == OPND_REG)
641 reg = r_opnd.as.reg.reg_no & 7;
642 else
643 reg = 0;
644
645 // Encode the rm field
646 uint8_t rm;
647 if (rm_opnd.type == OPND_REG)
648 {
649 rm = rm_opnd.as.reg.reg_no & 7;
650 }
651 else
652 {
653 if (need_sib)
654 rm = 4;
655 else
656 rm = rm_opnd.as.mem.base_reg_no & 7;
657 }
658
659 // Encode and write the ModR/M byte
660 uint8_t rm_byte = (mod << 6) + (reg << 3) + (rm);
661 cb_write_byte(cb, rm_byte);
662
663 // Add the SIB byte, if needed
664 if (need_sib)
665 {
666 // SIB.scale (2 bits)
667 // SIB.index (3 bits)
668 // SIB.base (3 bits)
669
670 assert (rm_opnd.type == OPND_MEM);
671
672 // Encode the scale value
673 uint8_t scale = rm_opnd.as.mem.scale_exp;
674
675 // Encode the index value
676 uint8_t index;
677 if (!rm_opnd.as.mem.has_idx)
678 index = 4;
679 else
680 index = rm_opnd.as.mem.idx_reg_no & 7;
681
682 // Encode the base register
683 uint8_t base = rm_opnd.as.mem.base_reg_no & 7;
684
685 // Encode and write the SIB byte
686 uint8_t sib_byte = (scale << 6) + (index << 3) + (base);
687 cb_write_byte(cb, sib_byte);
688 }
689
690 // Add the displacement
691 if (rm_opnd.type == OPND_MEM)
692 {
693 uint32_t dsize = disp_size(rm_opnd);
694 if (dsize > 0)
695 cb_write_int(cb, rm_opnd.as.mem.disp, dsize);
696 }
697}
698
699// Encode a mul-like single-operand RM instruction
700static void write_rm_unary(
701 codeblock_t *cb,
702 const char *mnem,
703 uint8_t opMemReg8,
704 uint8_t opMemRegPref,
705 uint8_t opExt,
706 x86opnd_t opnd)
707{
708 // Write a disassembly string
709 //cb.writeASM(mnem, opnd);
710
711 // Check the size of opnd0
712 uint32_t opndSize;
713 if (opnd.type == OPND_REG || opnd.type == OPND_MEM)
714 opndSize = opnd.num_bits;
715 else
716 rb_bug("yjit: invalid operand");
717
718 assert (opndSize == 8 || opndSize == 16 || opndSize == 32 || opndSize == 64);
719 bool szPref = opndSize == 16;
720 bool rexW = opndSize == 64;
721
722 if (opndSize == 8)
723 cb_write_rm(cb, false, false, NO_OPND, opnd, opExt, 1, opMemReg8);
724 else
725 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd, opExt, 1, opMemRegPref);
726}
727
728// Encode an add-like RM instruction with multiple possible encodings
729static void cb_write_rm_multi(
730 codeblock_t *cb,
731 const char *mnem,
732 uint8_t opMemReg8,
733 uint8_t opMemRegPref,
734 uint8_t opRegMem8,
735 uint8_t opRegMemPref,
736 uint8_t opMemImm8,
737 uint8_t opMemImmSml,
738 uint8_t opMemImmLrg,
739 uint8_t opExtImm,
740 x86opnd_t opnd0,
741 x86opnd_t opnd1)
742{
743 assert (opnd0.type == OPND_REG || opnd0.type == OPND_MEM);
744
745 /*
746 // Write disassembly string
747 if (!opnd1.isNone)
748 cb.writeASM(mnem, opnd0, opnd1);
749 else
750 cb.writeASM(mnem, opnd0);
751 */
752
753 // Check the size of opnd0
754 uint32_t opndSize = opnd0.num_bits;
755
756 // Check the size of opnd1
757 if (opnd1.type == OPND_REG || opnd1.type == OPND_MEM)
758 {
759 assert (opnd1.num_bits == opndSize && "operand size mismatch");
760 }
761 else if (opnd1.type == OPND_IMM)
762 {
763 assert (opnd1.num_bits <= opndSize);
764 }
765
766 assert (opndSize == 8 || opndSize == 16 || opndSize == 32 || opndSize == 64);
767 bool szPref = opndSize == 16;
768 bool rexW = opndSize == 64;
769
770 // R/M + Reg
771 if ((opnd0.type == OPND_MEM && opnd1.type == OPND_REG) ||
772 (opnd0.type == OPND_REG && opnd1.type == OPND_REG))
773 {
774 // R/M is opnd0
775 if (opndSize == 8)
776 cb_write_rm(cb, false, false, opnd1, opnd0, 0xFF, 1, opMemReg8);
777 else
778 cb_write_rm(cb, szPref, rexW, opnd1, opnd0, 0xFF, 1, opMemRegPref);
779 }
780
781 // Reg + R/M
782 else if (opnd0.type == OPND_REG && opnd1.type == OPND_MEM)
783 {
784 // R/M is opnd1
785 if (opndSize == 8)
786 cb_write_rm(cb, false, false, opnd0, opnd1, 0xFF, 1, opRegMem8);
787 else
788 cb_write_rm(cb, szPref, rexW, opnd0, opnd1, 0xFF, 1, opRegMemPref);
789 }
790
791 // R/M + Imm
792 else if (opnd1.type == OPND_IMM)
793 {
794 // 8-bit immediate
795 if (opnd1.num_bits <= 8)
796 {
797 if (opndSize == 8)
798 cb_write_rm(cb, false, false, NO_OPND, opnd0, opExtImm, 1, opMemImm8);
799 else
800 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExtImm, 1, opMemImmSml);
801
802 cb_write_int(cb, opnd1.as.imm, 8);
803 }
804
805 // 32-bit immediate
806 else if (opnd1.num_bits <= 32)
807 {
808 assert (opnd1.num_bits <= opndSize && "immediate too large for dst");
809 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExtImm, 1, opMemImmLrg);
810 cb_write_int(cb, opnd1.as.imm, (opndSize > 32)? 32:opndSize);
811 }
812
813 // Immediate too large
814 else
815 {
816 assert (false && "immediate value too large");
817 }
818 }
819
820 // Invalid operands
821 else
822 {
823 assert (false && "invalid operand combination");
824 }
825}
826
827// Encode a single-operand shift instruction
828static void cb_write_shift(
829 codeblock_t *cb,
830 const char *mnem,
831 uint8_t opMemOnePref,
832 uint8_t opMemClPref,
833 uint8_t opMemImmPref,
834 uint8_t opExt,
835 x86opnd_t opnd0,
836 x86opnd_t opnd1)
837{
838 // Write a disassembly string
839 //cb.writeASM(mnem, opnd0, opnd1);
840
841 // Check the size of opnd0
842 uint32_t opndSize;
843 if (opnd0.type == OPND_REG || opnd0.type == OPND_MEM)
844 opndSize = opnd0.num_bits;
845 else
846 rb_bug("yjit: shift: invalid first operand");
847
848 assert (opndSize == 16 || opndSize == 32 || opndSize == 64);
849 bool szPref = opndSize == 16;
850 bool rexW = opndSize == 64;
851
852 if (opnd1.type == OPND_IMM)
853 {
854 if (opnd1.as.imm == 1)
855 {
856 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExt, 1, opMemOnePref);
857 }
858 else
859 {
860 assert (opnd1.num_bits <= 8);
861 cb_write_rm(cb, szPref, rexW, NO_OPND, opnd0, opExt, 1, opMemImmPref);
862 cb_write_byte(cb, (uint8_t)opnd1.as.imm);
863 }
864 }
865 /*
866 else if (opnd1.isReg && opnd1.reg == CL)
867 {
868 cb.writeRMInstr!('l', opExt, opMemClPref)(szPref, rexW, opnd0, X86Opnd.NONE);
869 }
870 */
871 else
872 {
873 assert (false);
874 }
875}
876
877// Encode a relative jump to a label (direct or conditional)
878// Note: this always encodes a 32-bit offset
879static void cb_write_jcc(codeblock_t *cb, const char *mnem, uint8_t op0, uint8_t op1, uint32_t label_idx)
880{
881 //cb.writeASM(mnem, label);
882
883 // Write the opcode
884 if (op0 != 0xFF)
885 cb_write_byte(cb, op0);
886 cb_write_byte(cb, op1);
887
888 // Add a reference to the label
889 cb_label_ref(cb, label_idx);
890
891 // Relative 32-bit offset to be patched
892 cb_write_int(cb, 0, 32);
893}
894
895// Encode a relative jump to a pointer at a 32-bit offset (direct or conditional)
896static void cb_write_jcc_ptr(codeblock_t *cb, const char *mnem, uint8_t op0, uint8_t op1, uint8_t *dst_ptr)
897{
898 //cb.writeASM(mnem, label);
899
900 // Write the opcode
901 if (op0 != 0xFF)
902 cb_write_byte(cb, op0);
903 cb_write_byte(cb, op1);
904
905 // Pointer to the end of this jump instruction
906 uint8_t *end_ptr = cb_get_ptr(cb, cb->write_pos + 4);
907
908 // Compute the jump offset
909 int64_t rel64 = (int64_t)(dst_ptr - end_ptr);
910 if (rel64 >= INT32_MIN && rel64 <= INT32_MAX) {
911 // Write the relative 32-bit jump offset
912 cb_write_int(cb, (int32_t)rel64, 32);
913 }
914 else {
915 // Offset doesn't fit in 4 bytes. Report error.
916 cb->dropped_bytes = true;
917 }
918}
919
920// Encode a conditional move instruction
921static void cb_write_cmov(codeblock_t *cb, const char *mnem, uint8_t opcode1, x86opnd_t dst, x86opnd_t src)
922{
923 //cb.writeASM(mnem, dst, src);
924
925 assert (dst.type == OPND_REG);
926 assert (src.type == OPND_REG || src.type == OPND_MEM);
927 assert (dst.num_bits >= 16 && "invalid dst reg size in cmov");
928
929 bool szPref = dst.num_bits == 16;
930 bool rexW = dst.num_bits == 64;
931
932 cb_write_rm(cb, szPref, rexW, dst, src, 0xFF, 2, 0x0F, opcode1);
933}
934
935// add - Integer addition
936void add(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
937{
938 cb_write_rm_multi(
939 cb,
940 "add",
941 0x00, // opMemReg8
942 0x01, // opMemRegPref
943 0x02, // opRegMem8
944 0x03, // opRegMemPref
945 0x80, // opMemImm8
946 0x83, // opMemImmSml
947 0x81, // opMemImmLrg
948 0x00, // opExtImm
949 opnd0,
950 opnd1
951 );
952}
953
955void and(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
956{
957 cb_write_rm_multi(
958 cb,
959 "and",
960 0x20, // opMemReg8
961 0x21, // opMemRegPref
962 0x22, // opRegMem8
963 0x23, // opRegMemPref
964 0x80, // opMemImm8
965 0x83, // opMemImmSml
966 0x81, // opMemImmLrg
967 0x04, // opExtImm
968 opnd0,
969 opnd1
970 );
971}
972
973// call - Call to a pointer with a 32-bit displacement offset
974static void call_rel32(codeblock_t *cb, int32_t rel32)
975{
976 //cb.writeASM("call", rel32);
977
978 // Write the opcode
979 cb_write_byte(cb, 0xE8);
980
981 // Write the relative 32-bit jump offset
982 cb_write_int(cb, (int32_t)rel32, 32);
983}
984
985// call - Call a pointer, encode with a 32-bit offset if possible
986void call_ptr(codeblock_t *cb, x86opnd_t scratch_reg, uint8_t *dst_ptr)
987{
988 assert (scratch_reg.type == OPND_REG);
989
990 // Pointer to the end of this call instruction
991 uint8_t *end_ptr = cb_get_ptr(cb, cb->write_pos + 5);
992
993 // Compute the jump offset
994 int64_t rel64 = (int64_t)(dst_ptr - end_ptr);
995
996 // If the offset fits in 32-bit
997 if (rel64 >= INT32_MIN && rel64 <= INT32_MAX) {
998 call_rel32(cb, (int32_t)rel64);
999 return;
1000 }
1001
1002 // Move the pointer into the scratch register and call
1003 mov(cb, scratch_reg, const_ptr_opnd(dst_ptr));
1004 call(cb, scratch_reg);
1005}
1006
1008void call_label(codeblock_t *cb, uint32_t label_idx)
1009{
1010 //cb.writeASM("call", label);
1011
1012 // Write the opcode
1013 cb_write_byte(cb, 0xE8);
1014
1015 // Add a reference to the label
1016 cb_label_ref(cb, label_idx);
1017
1018 // Relative 32-bit offset to be patched
1019 cb_write_int(cb, 0, 32);
1020}
1021
1023void call(codeblock_t *cb, x86opnd_t opnd)
1024{
1025 //cb.writeASM("call", opnd);
1026 cb_write_rm(cb, false, false, NO_OPND, opnd, 2, 1, 0xFF);
1027}
1028
1030void cmova(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmova", 0x47, dst, src); }
1031void cmovae(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovae", 0x43, dst, src); }
1032void cmovb(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovb", 0x42, dst, src); }
1033void cmovbe(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovbe", 0x46, dst, src); }
1034void cmovc(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovc", 0x42, dst, src); }
1035void cmove(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmove", 0x44, dst, src); }
1036void cmovg(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovg", 0x4F, dst, src); }
1037void cmovge(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovge", 0x4D, dst, src); }
1038void cmovl(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovl", 0x4C, dst, src); }
1039void cmovle(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovle", 0x4E, dst, src); }
1040void cmovna(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovna", 0x46, dst, src); }
1041void cmovnae(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnae", 0x42, dst, src); }
1042void cmovnb(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnb", 0x43, dst, src); }
1043void cmovnbe(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnbe", 0x47, dst, src); }
1044void cmovnc(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnc", 0x43, dst, src); }
1045void cmovne(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovne", 0x45, dst, src); }
1046void cmovng(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovng", 0x4E, dst, src); }
1047void cmovnge(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnge", 0x4C, dst, src); }
1048void cmovnl(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnl" , 0x4D, dst, src); }
1049void cmovnle(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnle", 0x4F, dst, src); }
1050void cmovno(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovno", 0x41, dst, src); }
1051void cmovnp(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnp", 0x4B, dst, src); }
1052void cmovns(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovns", 0x49, dst, src); }
1053void cmovnz(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovnz", 0x45, dst, src); }
1054void cmovo(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovo", 0x40, dst, src); }
1055void cmovp(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovp", 0x4A, dst, src); }
1056void cmovpe(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovpe", 0x4A, dst, src); }
1057void cmovpo(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovpo", 0x4B, dst, src); }
1058void cmovs(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovs", 0x48, dst, src); }
1059void cmovz(codeblock_t *cb, x86opnd_t dst, x86opnd_t src) { cb_write_cmov(cb, "cmovz", 0x44, dst, src); }
1060
1062void cmp(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1063{
1064 cb_write_rm_multi(
1065 cb,
1066 "cmp",
1067 0x38, // opMemReg8
1068 0x39, // opMemRegPref
1069 0x3A, // opRegMem8
1070 0x3B, // opRegMemPref
1071 0x80, // opMemImm8
1072 0x83, // opMemImmSml
1073 0x81, // opMemImmLrg
1074 0x07, // opExtImm
1075 opnd0,
1076 opnd1
1077 );
1078}
1079
1081void cdq(codeblock_t *cb)
1082{
1083 //cb.writeASM("cdq");
1084 cb_write_byte(cb, 0x99);
1085}
1086
1088void cqo(codeblock_t *cb)
1089{
1090 //cb.writeASM("cqo");
1091 cb_write_bytes(cb, 2, 0x48, 0x99);
1092}
1093
1095void int3(codeblock_t *cb)
1096{
1097 //cb.writeASM("INT 3");
1098 cb_write_byte(cb, 0xCC);
1099}
1100
1101/*
1102// div - Unsigned integer division
1103alias div = writeRMUnary!(
1104 "div",
1105 0xF6, // opMemReg8
1106 0xF7, // opMemRegPref
1107 0x06 // opExt
1108);
1109*/
1110
1111/*
1113alias divsd = writeXMM64!(
1114 "divsd",
1115 0xF2, // prefix
1116 0x0F, // opRegMem0
1117 0x5E // opRegMem1
1118);
1119*/
1120
1121/*
1122// idiv - Signed integer division
1123alias idiv = writeRMUnary!(
1124 "idiv",
1125 0xF6, // opMemReg8
1126 0xF7, // opMemRegPref
1127 0x07 // opExt
1128);
1129*/
1130
1131/*
1133void imul(CodeBlock cb, X86Opnd opnd0, X86Opnd opnd1)
1134{
1135 cb.writeASM("imul", opnd0, opnd1);
1136
1137 assert (opnd0.isReg, "invalid first operand");
1138 auto opndSize = opnd0.reg.size;
1139
1140 // Check the size of opnd1
1141 if (opnd1.isReg)
1142 assert (opnd1.reg.size is opndSize, "operand size mismatch");
1143 else if (opnd1.isMem)
1144 assert (opnd1.mem.size is opndSize, "operand size mismatch");
1145
1146 assert (opndSize is 16 || opndSize is 32 || opndSize is 64);
1147 auto szPref = opndSize is 16;
1148 auto rexW = opndSize is 64;
1149
1150 cb.writeRMInstr!('r', 0xFF, 0x0F, 0xAF)(szPref, rexW, opnd0, opnd1);
1151}
1152*/
1153
1154/*
1156void imul(CodeBlock cb, X86Opnd opnd0, X86Opnd opnd1, X86Opnd opnd2)
1157{
1158 cb.writeASM("imul", opnd0, opnd1, opnd2);
1159
1160 assert (opnd0.isReg, "invalid first operand");
1161 auto opndSize = opnd0.reg.size;
1162
1163 // Check the size of opnd1
1164 if (opnd1.isReg)
1165 assert (opnd1.reg.size is opndSize, "operand size mismatch");
1166 else if (opnd1.isMem)
1167 assert (opnd1.mem.size is opndSize, "operand size mismatch");
1168
1169 assert (opndSize is 16 || opndSize is 32 || opndSize is 64);
1170 auto szPref = opndSize is 16;
1171 auto rexW = opndSize is 64;
1172
1173 assert (opnd2.isImm, "invalid third operand");
1174 auto imm = opnd2.imm;
1175
1176 // 8-bit immediate
1177 if (imm.immSize <= 8)
1178 {
1179 cb.writeRMInstr!('r', 0xFF, 0x6B)(szPref, rexW, opnd0, opnd1);
1180 cb.writeInt(imm.imm, 8);
1181 }
1182
1183 // 32-bit immediate
1184 else if (imm.immSize <= 32)
1185 {
1186 assert (imm.immSize <= opndSize, "immediate too large for dst");
1187 cb.writeRMInstr!('r', 0xFF, 0x69)(szPref, rexW, opnd0, opnd1);
1188 cb.writeInt(imm.imm, min(opndSize, 32));
1189 }
1190
1191 // Immediate too large
1192 else
1193 {
1194 assert (false, "immediate value too large");
1195 }
1196}
1197*/
1198
1200void ja_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "ja" , 0x0F, 0x87, label_idx); }
1201void jae_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jae" , 0x0F, 0x83, label_idx); }
1202void jb_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jb" , 0x0F, 0x82, label_idx); }
1203void jbe_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jbe" , 0x0F, 0x86, label_idx); }
1204void jc_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jc" , 0x0F, 0x82, label_idx); }
1205void je_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "je" , 0x0F, 0x84, label_idx); }
1206void jg_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jg" , 0x0F, 0x8F, label_idx); }
1207void jge_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jge" , 0x0F, 0x8D, label_idx); }
1208void jl_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jl" , 0x0F, 0x8C, label_idx); }
1209void jle_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jle" , 0x0F, 0x8E, label_idx); }
1210void jna_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jna" , 0x0F, 0x86, label_idx); }
1211void jnae_label(codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnae", 0x0F, 0x82, label_idx); }
1212void jnb_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnb" , 0x0F, 0x83, label_idx); }
1213void jnbe_label(codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnbe", 0x0F, 0x87, label_idx); }
1214void jnc_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnc" , 0x0F, 0x83, label_idx); }
1215void jne_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jne" , 0x0F, 0x85, label_idx); }
1216void jng_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jng" , 0x0F, 0x8E, label_idx); }
1217void jnge_label(codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnge", 0x0F, 0x8C, label_idx); }
1218void jnl_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnl" , 0x0F, 0x8D, label_idx); }
1219void jnle_label(codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnle", 0x0F, 0x8F, label_idx); }
1220void jno_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jno" , 0x0F, 0x81, label_idx); }
1221void jnp_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnp" , 0x0F, 0x8b, label_idx); }
1222void jns_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jns" , 0x0F, 0x89, label_idx); }
1223void jnz_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jnz" , 0x0F, 0x85, label_idx); }
1224void jo_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jo" , 0x0F, 0x80, label_idx); }
1225void jp_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jp" , 0x0F, 0x8A, label_idx); }
1226void jpe_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jpe" , 0x0F, 0x8A, label_idx); }
1227void jpo_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jpo" , 0x0F, 0x8B, label_idx); }
1228void js_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "js" , 0x0F, 0x88, label_idx); }
1229void jz_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jz" , 0x0F, 0x84, label_idx); }
1230void jmp_label (codeblock_t *cb, uint32_t label_idx) { cb_write_jcc(cb, "jmp" , 0xFF, 0xE9, label_idx); }
1231
1233void ja_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "ja" , 0x0F, 0x87, ptr); }
1234void jae_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jae" , 0x0F, 0x83, ptr); }
1235void jb_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jb" , 0x0F, 0x82, ptr); }
1236void jbe_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jbe" , 0x0F, 0x86, ptr); }
1237void jc_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jc" , 0x0F, 0x82, ptr); }
1238void je_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "je" , 0x0F, 0x84, ptr); }
1239void jg_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jg" , 0x0F, 0x8F, ptr); }
1240void jge_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jge" , 0x0F, 0x8D, ptr); }
1241void jl_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jl" , 0x0F, 0x8C, ptr); }
1242void jle_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jle" , 0x0F, 0x8E, ptr); }
1243void jna_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jna" , 0x0F, 0x86, ptr); }
1244void jnae_ptr(codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnae", 0x0F, 0x82, ptr); }
1245void jnb_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnb" , 0x0F, 0x83, ptr); }
1246void jnbe_ptr(codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnbe", 0x0F, 0x87, ptr); }
1247void jnc_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnc" , 0x0F, 0x83, ptr); }
1248void jne_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jne" , 0x0F, 0x85, ptr); }
1249void jng_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jng" , 0x0F, 0x8E, ptr); }
1250void jnge_ptr(codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnge", 0x0F, 0x8C, ptr); }
1251void jnl_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnl" , 0x0F, 0x8D, ptr); }
1252void jnle_ptr(codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnle", 0x0F, 0x8F, ptr); }
1253void jno_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jno" , 0x0F, 0x81, ptr); }
1254void jnp_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnp" , 0x0F, 0x8b, ptr); }
1255void jns_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jns" , 0x0F, 0x89, ptr); }
1256void jnz_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jnz" , 0x0F, 0x85, ptr); }
1257void jo_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jo" , 0x0F, 0x80, ptr); }
1258void jp_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jp" , 0x0F, 0x8A, ptr); }
1259void jpe_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jpe" , 0x0F, 0x8A, ptr); }
1260void jpo_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jpo" , 0x0F, 0x8B, ptr); }
1261void js_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "js" , 0x0F, 0x88, ptr); }
1262void jz_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jz" , 0x0F, 0x84, ptr); }
1263void jmp_ptr (codeblock_t *cb, uint8_t *ptr) { cb_write_jcc_ptr(cb, "jmp" , 0xFF, 0xE9, ptr); }
1264
1266void jmp_rm(codeblock_t *cb, x86opnd_t opnd)
1267{
1268 //cb.writeASM("jmp", opnd);
1269 cb_write_rm(cb, false, false, NO_OPND, opnd, 4, 1, 0xFF);
1270}
1271
1272// jmp - Jump with relative 32-bit offset
1273void jmp32(codeblock_t *cb, int32_t offset)
1274{
1275 //cb.writeASM("jmp", ((offset > 0)? "+":"-") ~ to!string(offset));
1276 cb_write_byte(cb, 0xE9);
1277 cb_write_int(cb, offset, 32);
1278}
1279
1281void lea(codeblock_t *cb, x86opnd_t dst, x86opnd_t src)
1282{
1283 //cb.writeASM("lea", dst, src);
1284 assert (dst.num_bits == 64);
1285 cb_write_rm(cb, false, true, dst, src, 0xFF, 1, 0x8D);
1286}
1287
1288// Does this number fit in 32 bits and stays the same if you zero extend it to 64 bit?
1289// If the sign bit is clear, sign extension and zero extension yield the same
1290// result.
1291static bool
1292zero_extendable_32bit(uint64_t number)
1293{
1294 return number <= UINT32_MAX && (number & (1ull << 31ull)) == 0;
1295}
1296
1298void mov(codeblock_t *cb, x86opnd_t dst, x86opnd_t src)
1299{
1300 // R/M + Imm
1301 if (src.type == OPND_IMM)
1302 {
1303 //cb.writeASM("mov", dst, src);
1304
1305 // R + Imm
1306 if (dst.type == OPND_REG)
1307 {
1308 assert (
1309 src.num_bits <= dst.num_bits ||
1310 unsig_imm_size(src.as.imm) <= dst.num_bits
1311 );
1312
1313 // In case the source immediate could be zero extended to be 64
1314 // bit, we can use the 32-bit operands version of the instruction.
1315 // For example, we can turn mov(rax, 0x34) into the equivalent
1316 // mov(eax, 0x34).
1317 if (dst.num_bits == 64 && zero_extendable_32bit(src.as.unsig_imm)) {
1318 if (rex_needed(dst))
1319 cb_write_rex(cb, false, 0, 0, dst.as.reg.reg_no);
1320 cb_write_opcode(cb, 0xB8, dst);
1321 cb_write_int(cb, src.as.imm, 32);
1322 }
1323 else {
1324 if (dst.num_bits == 16)
1325 cb_write_byte(cb, 0x66);
1326
1327 if (rex_needed(dst) || dst.num_bits == 64)
1328 cb_write_rex(cb, dst.num_bits == 64, 0, 0, dst.as.reg.reg_no);
1329
1330 cb_write_opcode(cb, (dst.num_bits == 8)? 0xB0:0xB8, dst);
1331
1332 cb_write_int(cb, src.as.imm, dst.num_bits);
1333 }
1334 }
1335
1336 // M + Imm
1337 else if (dst.type == OPND_MEM)
1338 {
1339 assert (src.num_bits <= dst.num_bits);
1340
1341 if (dst.num_bits == 8)
1342 cb_write_rm(cb, false, false, NO_OPND, dst, 0xFF, 1, 0xC6);
1343 else
1344 cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, NO_OPND, dst, 0, 1, 0xC7);
1345
1346 const uint32_t output_num_bits = (dst.num_bits > 32u) ? 32u : dst.num_bits;
1347 // assert that we can write whole immediate without loss of information
1348 assert (sig_imm_size(src.as.imm) <= output_num_bits);
1349 cb_write_int(cb, src.as.imm, output_num_bits);
1350 }
1351
1352 else
1353 {
1354 assert (false);
1355 }
1356 }
1357 else
1358 {
1359 cb_write_rm_multi(
1360 cb,
1361 "mov",
1362 0x88, // opMemReg8
1363 0x89, // opMemRegPref
1364 0x8A, // opRegMem8
1365 0x8B, // opRegMemPref
1366 0xC6, // opMemImm8
1367 0xFF, // opMemImmSml (not available)
1368 0xFF, // opMemImmLrg
1369 0xFF, // opExtImm
1370 dst,
1371 src
1372 );
1373 }
1374}
1375
1377void movsx(codeblock_t *cb, x86opnd_t dst, x86opnd_t src)
1378{
1379 assert (dst.type == OPND_REG);
1380 assert (src.type == OPND_REG || src.type == OPND_MEM);
1381 assert (src.num_bits < dst.num_bits);
1382
1383 //cb.writeASM("movsx", dst, src);
1384
1385 if (src.num_bits == 8)
1386 {
1387 cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, dst, src, 0xFF, 2, 0x0F, 0xBE);
1388 }
1389 else if (src.num_bits == 16)
1390 {
1391 cb_write_rm(cb, dst.num_bits == 16, dst.num_bits == 64, dst, src, 0xFF, 2, 0x0F, 0xBF);
1392 }
1393 else if (src.num_bits == 32)
1394 {
1395 cb_write_rm(cb, false, true, dst, src, 0xFF, 1, 0x63);
1396 }
1397 else
1398 {
1399 assert (false);
1400 }
1401}
1402
1403/*
1405void movzx(codeblock_t *cb, x86opnd_t dst, x86opnd_t src)
1406{
1407 cb.writeASM("movzx", dst, src);
1408
1409 uint32_t dstSize;
1410 if (dst.isReg)
1411 dstSize = dst.reg.size;
1412 else
1413 assert (false, "movzx dst must be a register");
1414
1415 uint32_t srcSize;
1416 if (src.isReg)
1417 srcSize = src.reg.size;
1418 else if (src.isMem)
1419 srcSize = src.mem.size;
1420 else
1421 assert (false);
1422
1423 assert (
1424 srcSize < dstSize,
1425 "movzx: srcSize >= dstSize"
1426 );
1427
1428 if (srcSize is 8)
1429 {
1430 cb.writeRMInstr!('r', 0xFF, 0x0F, 0xB6)(dstSize is 16, dstSize is 64, dst, src);
1431 }
1432 else if (srcSize is 16)
1433 {
1434 cb.writeRMInstr!('r', 0xFF, 0x0F, 0xB7)(dstSize is 16, dstSize is 64, dst, src);
1435 }
1436 else
1437 {
1438 assert (false, "invalid src operand size for movxz");
1439 }
1440}
1441*/
1442
1443// neg - Integer negation (multiplication by -1)
1444void neg(codeblock_t *cb, x86opnd_t opnd)
1445{
1446 write_rm_unary(
1447 cb,
1448 "neg",
1449 0xF6, // opMemReg8
1450 0xF7, // opMemRegPref
1451 0x03, // opExt
1452 opnd
1453 );
1454}
1455
1456// nop - Noop, one or multiple bytes long
1457void nop(codeblock_t *cb, uint32_t length)
1458{
1459 switch (length) {
1460 case 0:
1461 break;
1462
1463 case 1:
1464 //cb.writeASM("nop1");
1465 cb_write_byte(cb, 0x90);
1466 break;
1467
1468 case 2:
1469 //cb.writeASM("nop2");
1470 cb_write_bytes(cb, 2, 0x66,0x90);
1471 break;
1472
1473 case 3:
1474 //cb.writeASM("nop3");
1475 cb_write_bytes(cb, 3, 0x0F,0x1F,0x00);
1476 break;
1477
1478 case 4:
1479 //cb.writeASM("nop4");
1480 cb_write_bytes(cb, 4, 0x0F,0x1F,0x40,0x00);
1481 break;
1482
1483 case 5:
1484 //cb.writeASM("nop5");
1485 cb_write_bytes(cb, 5, 0x0F,0x1F,0x44,0x00,0x00);
1486 break;
1487
1488 case 6:
1489 //cb.writeASM("nop6");
1490 cb_write_bytes(cb, 6, 0x66,0x0F,0x1F,0x44,0x00,0x00);
1491 break;
1492
1493 case 7:
1494 //cb.writeASM("nop7");
1495 cb_write_bytes(cb, 7, 0x0F,0x1F,0x80,0x00,0x00,0x00,0x00);
1496 break;
1497
1498 case 8:
1499 //cb.writeASM("nop8");
1500 cb_write_bytes(cb, 8, 0x0F,0x1F,0x84,0x00,0x00,0x00,0x00,0x00);
1501 break;
1502
1503 case 9:
1504 //cb.writeASM("nop9");
1505 cb_write_bytes(cb, 9, 0x66,0x0F,0x1F,0x84,0x00,0x00,0x00,0x00,0x00);
1506 break;
1507
1508 default:
1509 {
1510 uint32_t written = 0;
1511 while (written + 9 <= length)
1512 {
1513 nop(cb, 9);
1514 written += 9;
1515 }
1516 nop(cb, length - written);
1517 }
1518 break;
1519 }
1520}
1521
1522// not - Bitwise NOT
1523void not(codeblock_t *cb, x86opnd_t opnd)
1524{
1525 write_rm_unary(
1526 cb,
1527 "not",
1528 0xF6, // opMemReg8
1529 0xF7, // opMemRegPref
1530 0x02, // opExt
1531 opnd
1532 );
1533}
1534
1536void or(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1537{
1538 cb_write_rm_multi(
1539 cb,
1540 "or",
1541 0x08, // opMemReg8
1542 0x09, // opMemRegPref
1543 0x0A, // opRegMem8
1544 0x0B, // opRegMemPref
1545 0x80, // opMemImm8
1546 0x83, // opMemImmSml
1547 0x81, // opMemImmLrg
1548 0x01, // opExtImm
1549 opnd0,
1550 opnd1
1551 );
1552}
1553
1555void pop(codeblock_t *cb, x86opnd_t opnd)
1556{
1557 assert (opnd.num_bits == 64);
1558
1559 //cb.writeASM("pop", opnd);
1560
1561 if (opnd.type == OPND_REG) {
1562 if (rex_needed(opnd))
1563 cb_write_rex(cb, false, 0, 0, opnd.as.reg.reg_no);
1564 cb_write_opcode(cb, 0x58, opnd);
1565 }
1566 else if (opnd.type == OPND_MEM) {
1567 cb_write_rm(cb, false, false, NO_OPND, opnd, 0, 1, 0x8F);
1568 }
1569 else {
1570 assert(false && "unexpected operand type");
1571 }
1572}
1573
1575void popfq(codeblock_t *cb)
1576{
1577 //cb.writeASM("popfq");
1578
1579 // REX.W + 0x9D
1580 cb_write_bytes(cb, 2, 0x48, 0x9D);
1581}
1582
1584void push(codeblock_t *cb, x86opnd_t opnd)
1585{
1586 assert (opnd.num_bits == 64);
1587
1588 //cb.writeASM("push", opnd);
1589
1590 if (opnd.type == OPND_REG) {
1591 if (rex_needed(opnd))
1592 cb_write_rex(cb, false, 0, 0, opnd.as.reg.reg_no);
1593 cb_write_opcode(cb, 0x50, opnd);
1594 }
1595 else if (opnd.type == OPND_MEM) {
1596 cb_write_rm(cb, false, false, NO_OPND, opnd, 6, 1, 0xFF);
1597 }
1598 else {
1599 assert(false && "unexpected operand type");
1600 }
1601}
1602
1604void pushfq(codeblock_t *cb)
1605{
1606 //cb.writeASM("pushfq");
1607 cb_write_byte(cb, 0x9C);
1608}
1609
1611void ret(codeblock_t *cb)
1612{
1613 //cb.writeASM("ret");
1614 cb_write_byte(cb, 0xC3);
1615}
1616
1617// sal - Shift arithmetic left
1618void sal(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1619{
1620 cb_write_shift(
1621 cb,
1622 "sal",
1623 0xD1, // opMemOnePref,
1624 0xD3, // opMemClPref,
1625 0xC1, // opMemImmPref,
1626 0x04,
1627 opnd0,
1628 opnd1
1629 );
1630}
1631
1633void sar(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1634{
1635 cb_write_shift(
1636 cb,
1637 "sar",
1638 0xD1, // opMemOnePref,
1639 0xD3, // opMemClPref,
1640 0xC1, // opMemImmPref,
1641 0x07,
1642 opnd0,
1643 opnd1
1644 );
1645}
1646// shl - Shift logical left
1647void shl(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1648{
1649 cb_write_shift(
1650 cb,
1651 "shl",
1652 0xD1, // opMemOnePref,
1653 0xD3, // opMemClPref,
1654 0xC1, // opMemImmPref,
1655 0x04,
1656 opnd0,
1657 opnd1
1658 );
1659}
1660
1662void shr(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1663{
1664 cb_write_shift(
1665 cb,
1666 "shr",
1667 0xD1, // opMemOnePref,
1668 0xD3, // opMemClPref,
1669 0xC1, // opMemImmPref,
1670 0x05,
1671 opnd0,
1672 opnd1
1673 );
1674}
1675
1677void sub(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1678{
1679 cb_write_rm_multi(
1680 cb,
1681 "sub",
1682 0x28, // opMemReg8
1683 0x29, // opMemRegPref
1684 0x2A, // opRegMem8
1685 0x2B, // opRegMemPref
1686 0x80, // opMemImm8
1687 0x83, // opMemImmSml
1688 0x81, // opMemImmLrg
1689 0x05, // opExtImm
1690 opnd0,
1691 opnd1
1692 );
1693}
1694
1696void test(codeblock_t *cb, x86opnd_t rm_opnd, x86opnd_t test_opnd)
1697{
1698 assert (rm_opnd.type == OPND_REG || rm_opnd.type == OPND_MEM);
1699 assert (test_opnd.type == OPND_REG || test_opnd.type == OPND_IMM);
1700
1701 // If the second operand is an immediate
1702 if (test_opnd.type == OPND_IMM)
1703 {
1704 x86opnd_t imm_opnd = test_opnd;
1705
1706 if (imm_opnd.as.imm >= 0)
1707 {
1708 assert (unsig_imm_size(imm_opnd.as.unsig_imm) <= 32);
1709 assert (unsig_imm_size(imm_opnd.as.unsig_imm) <= rm_opnd.num_bits);
1710
1711 // Use the smallest operand size possible
1712 rm_opnd = resize_opnd(rm_opnd, unsig_imm_size(imm_opnd.as.unsig_imm));
1713
1714 if (rm_opnd.num_bits == 8)
1715 {
1716 cb_write_rm(cb, false, false, NO_OPND, rm_opnd, 0x00, 1, 0xF6);
1717 cb_write_int(cb, imm_opnd.as.imm, rm_opnd.num_bits);
1718 }
1719 else
1720 {
1721 cb_write_rm(cb, rm_opnd.num_bits == 16, false, NO_OPND, rm_opnd, 0x00, 1, 0xF7);
1722 cb_write_int(cb, imm_opnd.as.imm, rm_opnd.num_bits);
1723 }
1724 }
1725 else
1726 {
1727 // This mode only applies to 64-bit R/M operands with 32-bit signed immediates
1728 assert (imm_opnd.as.imm < 0);
1729 assert (sig_imm_size(imm_opnd.as.imm) <= 32);
1730 assert (rm_opnd.num_bits == 64);
1731 cb_write_rm(cb, false, true, NO_OPND, rm_opnd, 0x00, 1, 0xF7);
1732 cb_write_int(cb, imm_opnd.as.imm, 32);
1733 }
1734 }
1735 else
1736 {
1737 assert (test_opnd.num_bits == rm_opnd.num_bits);
1738
1739 if (rm_opnd.num_bits == 8)
1740 {
1741 cb_write_rm(cb, false, false, test_opnd, rm_opnd, 0xFF, 1, 0x84);
1742 }
1743 else
1744 {
1745 cb_write_rm(cb, rm_opnd.num_bits == 16, rm_opnd.num_bits == 64, test_opnd, rm_opnd, 0xFF, 1, 0x85);
1746 }
1747 }
1748}
1749
1751void ud2(codeblock_t *cb)
1752{
1753 cb_write_bytes(cb, 2, 0x0F, 0x0B);
1754}
1755
1757void xchg(codeblock_t *cb, x86opnd_t rm_opnd, x86opnd_t r_opnd)
1758{
1759 assert (rm_opnd.num_bits == 64);
1760 assert (r_opnd.num_bits == 64);
1761 assert (rm_opnd.type == OPND_REG);
1762 assert (r_opnd.type == OPND_REG);
1763
1764 // If we're exchanging with RAX
1765 if (rm_opnd.type == OPND_REG && rm_opnd.as.reg.reg_no == RAX.as.reg.reg_no)
1766 {
1767 // Write the REX byte
1768 cb_write_rex(cb, rm_opnd.num_bits == 64, 0, 0, r_opnd.as.reg.reg_no);
1769
1770 // Write the opcode and register number
1771 cb_write_byte(cb, 0x90 + (r_opnd.as.reg.reg_no & 7));
1772 }
1773 else
1774 {
1775 cb_write_rm(cb, rm_opnd.num_bits == 16, rm_opnd.num_bits == 64, r_opnd, rm_opnd, 0xFF, 1, 0x87);
1776 }
1777}
1778
1780void xor(codeblock_t *cb, x86opnd_t opnd0, x86opnd_t opnd1)
1781{
1782 cb_write_rm_multi(
1783 cb,
1784 "xor",
1785 0x30, // opMemReg8
1786 0x31, // opMemRegPref
1787 0x32, // opRegMem8
1788 0x33, // opRegMemPref
1789 0x80, // opMemImm8
1790 0x83, // opMemImmSml
1791 0x81, // opMemImmLrg
1792 0x06, // opExtImm
1793 opnd0,
1794 opnd1
1795 );
1796}
1797
1798// LOCK - lock prefix for atomic shared memory operations
1799void cb_write_lock_prefix(codeblock_t *cb)
1800{
1801 cb_write_byte(cb, 0xF0);
1802}
1803
1804void cb_mark_all_writeable(codeblock_t * cb)
1805{
1806 if (mprotect(cb->mem_block_, cb->mem_size, PROT_READ | PROT_WRITE)) {
1807 fprintf(stderr, "Couldn't make JIT page (%p) writeable, errno: %s", (void *)cb->mem_block_, strerror(errno));
1808 abort();
1809 }
1810}
1811
1812void cb_mark_position_writeable(codeblock_t * cb, uint32_t write_pos)
1813{
1814 uint32_t pagesize = (uint32_t)sysconf(_SC_PAGESIZE);
1815 uint32_t aligned_position = (write_pos / pagesize) * pagesize;
1816
1817 if (cb->current_aligned_write_pos != aligned_position) {
1818 cb->current_aligned_write_pos = aligned_position;
1819 void *const page_addr = cb_get_ptr(cb, aligned_position);
1820 if (mprotect(page_addr, pagesize, PROT_READ | PROT_WRITE)) {
1821 fprintf(stderr, "Couldn't make JIT page (%p) writeable, errno: %s", page_addr, strerror(errno));
1822 abort();
1823 }
1824 }
1825}
1826
1827void cb_mark_all_executable(codeblock_t * cb)
1828{
1829 cb->current_aligned_write_pos = ALIGNED_WRITE_POSITION_NONE;
1830 if (mprotect(cb->mem_block_, cb->mem_size, PROT_READ | PROT_EXEC)) {
1831 fprintf(stderr, "Couldn't make JIT page (%p) executable, errno: %s", (void *)cb->mem_block_, strerror(errno));
1832 abort();
1833 }
1834}
1835
void rb_bug(const char *fmt,...)
Interpreter panic switch.
Definition: error.c:802
Defines old _.
uint8_t base_reg_no
Base register number.
Definition: yjit_asm.h:102
bool has_idx
Has index register flag.
Definition: yjit_asm.h:111
bool is_iprel
IP-relative addressing flag.
Definition: yjit_asm.h:115
uint8_t scale_exp
SIB scale exponent value (power of two, two bits)
Definition: yjit_asm.h:108
int32_t disp
Constant displacement from the base, not scaled.
Definition: yjit_asm.h:118
uint8_t idx_reg_no
Index register number.
Definition: yjit_asm.h:105