14#include "ruby/internal/config.h"
21#include "internal/class.h"
22#include "internal/cont.h"
23#include "internal/file.h"
24#include "internal/hash.h"
25#include "internal/warnings.h"
28#include "mjit_worker.c"
30extern int rb_thread_create_mjit_thread(
void (*worker_func)(
void));
36get_uniq_filename(
unsigned long id,
const char *prefix,
const char *suffix)
38 char buff[70], *str = buff;
39 int size = sprint_uniq_filename(buff,
sizeof(buff),
id, prefix, suffix);
43 if (size <= (
int)
sizeof(buff)) {
44 memcpy(str, buff, size);
47 sprint_uniq_filename(str, size,
id, prefix, suffix);
55mjit_gc_start_hook(
void)
59 CRITICAL_SECTION_START(4,
"mjit_gc_start_hook");
61 verbose(4,
"Waiting wakeup from a worker for GC");
63 verbose(4,
"Getting wakeup from a worker for GC");
66 CRITICAL_SECTION_FINISH(4,
"mjit_gc_start_hook");
72mjit_gc_exit_hook(
void)
76 CRITICAL_SECTION_START(4,
"mjit_gc_exit_hook");
80 verbose(4,
"Sending wakeup signal to workers after GC");
83 CRITICAL_SECTION_FINISH(4,
"mjit_gc_exit_hook");
88mjit_cancel_all(
const char *reason)
94 if (mjit_opts.warnings || mjit_opts.verbose) {
95 fprintf(stderr,
"JIT cancel: Disabled JIT-ed code because %s\n", reason);
101mjit_update_references(
const rb_iseq_t *iseq)
106 CRITICAL_SECTION_START(4,
"mjit_update_references");
107 if (iseq->body->jit_unit) {
112 mjit_cancel_all(
"GC.compact is used");
119 list_for_each(&stale_units.head, unit, unode) {
120 if (unit->iseq == iseq) {
124 CRITICAL_SECTION_FINISH(4,
"mjit_update_references");
135 CRITICAL_SECTION_START(4,
"mjit_free_iseq");
138 if (iseq->body->jit_unit) {
141 iseq->body->jit_unit->iseq = NULL;
147 list_for_each(&stale_units.head, unit, unode) {
148 if (unit->iseq == iseq) {
152 CRITICAL_SECTION_FINISH(4,
"mjit_free_iseq");
163 list_for_each_safe(&list->head, unit, next, unode) {
164 list_del(&unit->unode);
165 if (!close_handle_p) unit->handle = NULL;
167 if (list == &stale_units) {
172 if (unit->handle && dlclose(unit->handle)) {
173 mjit_warning(
"failed to close handle for u%d: %s", unit->id, dlerror());
175 clean_temp_files(unit);
195 cont = calloc(1,
sizeof(
struct mjit_cont));
200 CRITICAL_SECTION_START(3,
"in mjit_cont_new");
201 if (first_cont == NULL) {
202 cont->next = cont->prev = NULL;
206 cont->next = first_cont;
207 first_cont->prev = cont;
210 CRITICAL_SECTION_FINISH(3,
"in mjit_cont_new");
219 CRITICAL_SECTION_START(3,
"in mjit_cont_new");
220 if (cont == first_cont) {
221 first_cont = cont->next;
222 if (first_cont != NULL)
223 first_cont->prev = NULL;
226 cont->prev->next = cont->next;
227 if (cont->next != NULL)
228 cont->next->prev = cont->prev;
230 CRITICAL_SECTION_FINISH(3,
"in mjit_cont_new");
241 for (cont = first_cont; cont != NULL; cont = next) {
257 unit->id = current_unit_num++;
259 iseq->body->jit_unit = unit;
266 return (body->type == ISEQ_TYPE_METHOD || body->type == ISEQ_TYPE_BLOCK)
267 && !body->builtin_inline_p;
275 if (!mjit_enabled || pch_status == PCH_FAILED)
277 if (!mjit_target_iseq_p(iseq->body)) {
278 iseq->body->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
283 CRITICAL_SECTION_START(3,
"in add_iseq_to_process");
286 if (rb_multi_ractor_p() && (uintptr_t)iseq->body->jit_func != NOT_ADDED_JIT_ISEQ_FUNC) {
287 CRITICAL_SECTION_FINISH(3,
"in add_iseq_to_process");
292 RB_DEBUG_COUNTER_INC(mjit_add_iseq_to_process);
293 iseq->body->jit_func = (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC;
295 if (iseq->body->jit_unit == NULL)
298 if (compile_info != NULL)
299 iseq->body->jit_unit->compile_info = *compile_info;
300 add_to_list(iseq->body->jit_unit, &unit_queue);
301 if (active_units.length >= mjit_opts.max_cache_size) {
306 verbose(3,
"Sending wakeup signal to workers in mjit_add_iseq_to_process");
308 CRITICAL_SECTION_FINISH(3,
"in add_iseq_to_process");
315rb_mjit_add_iseq_to_process(
const rb_iseq_t *iseq)
317 mjit_add_iseq_to_process(iseq, NULL,
false);
321#define MJIT_WAIT_TIMEOUT_SECONDS 60
330 while (body->jit_func == (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC) {
332 if (tries / 1000 > MJIT_WAIT_TIMEOUT_SECONDS || pch_status == PCH_FAILED) {
333 CRITICAL_SECTION_START(3,
"in rb_mjit_wait_call to set jit_func");
334 body->jit_func = (mjit_func_t)NOT_COMPILED_JIT_ISEQ_FUNC;
335 CRITICAL_SECTION_FINISH(3,
"in rb_mjit_wait_call to set jit_func");
336 mjit_warning(
"timed out to wait for JIT finish");
340 CRITICAL_SECTION_START(3,
"in rb_mjit_wait_call for a client wakeup");
342 CRITICAL_SECTION_FINISH(3,
"in rb_mjit_wait_call for a client wakeup");
356 if ((uintptr_t)body->jit_func <= (uintptr_t)LAST_JIT_ISEQ_FUNC) {
359 return body->jit_func(ec, ec->cfp);
365 assert(body->jit_unit != NULL);
366 return &body->jit_unit->compile_info;
372 if ((uintptr_t)iseq->body->jit_func <= (uintptr_t)LAST_JIT_ISEQ_FUNC)
375 verbose(1,
"JIT recompile: %s@%s:%d",
RSTRING_PTR(iseq->body->location.label),
377 assert(iseq->body->jit_unit != NULL);
379 if (UNLIKELY(mjit_opts.wait)) {
380 CRITICAL_SECTION_START(3,
"in rb_mjit_recompile_iseq");
381 remove_from_list(iseq->body->jit_unit, &active_units);
382 add_to_list(iseq->body->jit_unit, &stale_units);
383 mjit_add_iseq_to_process(iseq, &iseq->body->jit_unit->compile_info,
true);
384 CRITICAL_SECTION_FINISH(3,
"in rb_mjit_recompile_iseq");
385 mjit_wait(iseq->body);
391 CRITICAL_SECTION_START(3,
"in rb_mjit_recompile_iseq");
392 iseq->body->jit_unit->stale_p =
true;
393 iseq->body->jit_func = (mjit_func_t)NOT_READY_JIT_ISEQ_FUNC;
394 pending_stale_p =
true;
395 CRITICAL_SECTION_FINISH(3,
"in rb_mjit_recompile_iseq");
401rb_mjit_recompile_send(
const rb_iseq_t *iseq)
403 rb_mjit_iseq_compile_info(iseq->body)->disable_send_cache =
true;
404 mjit_recompile(iseq);
409rb_mjit_recompile_ivar(
const rb_iseq_t *iseq)
411 rb_mjit_iseq_compile_info(iseq->body)->disable_ivar_cache =
true;
412 mjit_recompile(iseq);
417rb_mjit_recompile_exivar(
const rb_iseq_t *iseq)
419 rb_mjit_iseq_compile_info(iseq->body)->disable_exivar_cache =
true;
420 mjit_recompile(iseq);
425rb_mjit_recompile_inlining(
const rb_iseq_t *iseq)
427 rb_mjit_iseq_compile_info(iseq->body)->disable_inlining =
true;
428 mjit_recompile(iseq);
433rb_mjit_recompile_const(
const rb_iseq_t *iseq)
435 rb_mjit_iseq_compile_info(iseq->body)->disable_const_cache =
true;
436 mjit_recompile(iseq);
439extern VALUE ruby_archlibdir_path, ruby_prefix_path;
443init_header_filename(
void)
450 const char *basedir =
"";
454 static const char libpathflag[] =
461 const size_t libpathflag_len =
sizeof(libpathflag) - 1;
465 basedir_val = ruby_prefix_path;
469 if (getenv(
"MJIT_SEARCH_BUILD_DIR")) {
475 const char *hdr = dlsym(RTLD_DEFAULT,
"MJIT_HEADER");
477 verbose(1,
"No MJIT_HEADER");
479 else if (hdr[0] !=
'/') {
480 verbose(1,
"Non-absolute header file path: %s", hdr);
482 else if (
stat(hdr, &st) || !S_ISREG(st.st_mode)) {
483 verbose(1,
"Non-file header file path: %s", hdr);
485 else if ((st.st_uid != getuid()) || (st.st_mode & 022) ||
487 verbose(1,
"Unsafe header file: uid=%ld mode=%#o %s",
488 (
long)st.st_uid, (
unsigned)st.st_mode, hdr);
494 verbose(3,
"PRELOADENV("PRELOADENV
")=%s", getenv(PRELOADENV));
496 unsetenv(PRELOADENV);
497 verbose(3,
"MJIT_HEADER: %s", hdr);
499 if (!header_file)
return false;
507 static const char header_name[] = MJIT_HEADER_INSTALL_DIR
"/" MJIT_MIN_HEADER_NAME;
508 const size_t header_name_len =
sizeof(header_name) - 1;
510 header_file =
xmalloc(baselen + header_name_len + 1);
511 p = append_str2(header_file, basedir, baselen);
512 p = append_str2(p, header_name, header_name_len + 1);
515 verbose(1,
"Cannot access header file: %s", header_file);
523 pch_file = get_uniq_filename(0, MJIT_TMP_PREFIX
"h",
".h.gch");
526 static const char pch_name[] = MJIT_HEADER_INSTALL_DIR
"/" MJIT_PRECOMPILED_HEADER_NAME;
527 const size_t pch_name_len =
sizeof(pch_name) - 1;
529 pch_file =
xmalloc(baselen + pch_name_len + 1);
530 p = append_str2(pch_file, basedir, baselen);
531 p = append_str2(p, pch_name, pch_name_len + 1);
533 verbose(1,
"Cannot access precompiled header file: %s", pch_file);
543 basedir_val = ruby_archlibdir_path;
546 libruby_pathflag = p =
xmalloc(libpathflag_len + baselen + 1);
547 p = append_str(p, libpathflag);
548 p = append_str2(p, basedir, baselen);
556UINT rb_w32_system_tmpdir(WCHAR *path, UINT len);
560system_default_tmpdir(
void)
564 WCHAR tmppath[_MAX_PATH];
565 UINT len = rb_w32_system_tmpdir(tmppath, numberof(tmppath));
567 int blen = WideCharToMultiByte(CP_UTF8, 0, tmppath, len, NULL, 0, NULL, NULL);
568 char *tmpdir =
xmalloc(blen + 1);
569 WideCharToMultiByte(CP_UTF8, 0, tmppath, len, tmpdir, blen, NULL, NULL);
573#elif defined _CS_DARWIN_USER_TEMP_DIR
574 char path[MAXPATHLEN];
575 size_t len = confstr(_CS_DARWIN_USER_TEMP_DIR, path,
sizeof(path));
578 if (len >
sizeof(path)) {
579 confstr(_CS_DARWIN_USER_TEMP_DIR, tmpdir, len);
582 memcpy(tmpdir, path, len);
591check_tmpdir(
const char *dir)
595 if (!dir)
return FALSE;
596 if (
stat(dir, &st))
return FALSE;
598# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
600 if (!S_ISDIR(st.st_mode))
return FALSE;
605 if (st.st_mode & S_IWOTH) {
607 if (!(st.st_mode & S_ISVTX))
return FALSE;
612 if (access(dir, W_OK))
return FALSE;
621# define RETURN_ENV(name) \
622 if (check_tmpdir(tmpdir = getenv(name))) return ruby_strdup(tmpdir)
623 RETURN_ENV(
"TMPDIR");
625 tmpdir = system_default_tmpdir();
626 if (check_tmpdir(tmpdir))
return tmpdir;
632#define MIN_CACHE_SIZE 10
634#define DEFAULT_MAX_CACHE_SIZE 10000
636#define DEFAULT_MIN_CALLS_TO_ADD 10000
642 stop_worker_p =
false;
643 worker_stopped =
false;
645 if (!rb_thread_create_mjit_thread(mjit_worker)) {
646 mjit_enabled =
false;
652 verbose(1,
"Failure in MJIT thread initialization\n");
660ruby_strndup(
const char *str,
size_t n)
671split_flags(
const char *flags)
673 char *buf[MAXPATHLEN];
676 for (; flags != NULL; flags = next) {
677 next = strchr(flags,
' ');
679 if (strlen(flags) > 0)
684 buf[i++] = ruby_strndup(flags, next - flags);
689 char **ret =
xmalloc(
sizeof(
char *) * (i + 1));
690 memcpy(ret, buf,
sizeof(
char *) * i);
706 if (mjit_opts.min_calls == 0)
707 mjit_opts.min_calls = DEFAULT_MIN_CALLS_TO_ADD;
708 if (mjit_opts.max_cache_size <= 0)
709 mjit_opts.max_cache_size = DEFAULT_MAX_CACHE_SIZE;
710 if (mjit_opts.max_cache_size < MIN_CACHE_SIZE)
711 mjit_opts.max_cache_size = MIN_CACHE_SIZE;
715 pch_status = PCH_SUCCESS;
717 pch_status = PCH_NOT_READY;
719 cc_path = CC_COMMON_ARGS[0];
720 verbose(2,
"MJIT: CC defaults to %s", cc_path);
721 cc_common_args =
xmalloc(
sizeof(CC_COMMON_ARGS));
722 memcpy((
void *)cc_common_args, CC_COMMON_ARGS,
sizeof(CC_COMMON_ARGS));
723 cc_added_args = split_flags(opts->debug_flags);
724 xfree(opts->debug_flags);
727 for (
size_t i = 0, j = 0; i <
sizeof(CC_COMMON_ARGS) /
sizeof(
char *); i++) {
728 if (CC_COMMON_ARGS[i] && strncmp(
"-save-temps", CC_COMMON_ARGS[i], strlen(
"-save-temps")) == 0)
730 cc_common_args[j] = CC_COMMON_ARGS[i];
735 tmp_dir = system_tmpdir();
736 verbose(2,
"MJIT: tmp_dir is %s", tmp_dir);
738 if (!init_header_filename()) {
739 mjit_enabled =
false;
740 verbose(1,
"Failure in MJIT header file name initialization\n");
743 pch_owner_pid = getpid();
757 rb_fiber_init_mjit_cont(GET_EC()->fiber_ptr);
768 while (!worker_stopped) {
769 verbose(3,
"Sending cancel signal to worker");
770 CRITICAL_SECTION_START(3,
"in stop_worker");
771 stop_worker_p =
true;
773 CRITICAL_SECTION_FINISH(3,
"in stop_worker");
774 RUBY_VM_CHECK_INTS(ec);
780mjit_pause(
bool wait_p)
783 rb_raise(rb_eRuntimeError,
"MJIT is not enabled");
785 if (worker_stopped) {
795 while (unit_queue.length > 0 && active_units.length < mjit_opts.max_cache_size) {
796 CRITICAL_SECTION_START(3,
"in mjit_pause for a worker wakeup");
798 CRITICAL_SECTION_FINISH(3,
"in mjit_pause for a worker wakeup");
812 rb_raise(rb_eRuntimeError,
"MJIT is not enabled");
814 if (!worker_stopped) {
818 if (!start_worker()) {
819 rb_raise(rb_eRuntimeError,
"Failed to resume MJIT worker");
831 list_for_each_safe(&list->head, unit, next, unode) {
833 if (unit->so_file) unit->so_file = NULL;
854mjit_child_after_fork(
void)
861 skip_cleaning_object_files(&active_units);
868#define MJIT_COUNTER 0
871mjit_dump_total_calls(
void)
874 fprintf(stderr,
"[MJIT_COUNTER] total_calls of active_units:\n");
875 list_for_each(&active_units.head, unit, unode) {
877 fprintf(stderr,
"%8ld: %s@%s:%d\n", iseq->body->total_calls,
RSTRING_PTR(iseq->body->location.label),
890mjit_finish(
bool close_handle_p)
896 verbose(2,
"Stopping worker thread");
897 CRITICAL_SECTION_START(3,
"in mjit_finish to wakeup from pch");
903 while (pch_status == PCH_NOT_READY) {
904 verbose(3,
"Waiting wakeup from make_pch");
907 CRITICAL_SECTION_FINISH(3,
"in mjit_finish to wakeup from pch");
919 mjit_dump_total_calls();
923 if (!mjit_opts.save_temps && getpid() == pch_owner_pid)
924 remove_file(pch_file);
926 xfree(header_file); header_file = NULL;
928 xfree((
void *)cc_common_args); cc_common_args = NULL;
929 for (
char **flag = cc_added_args; *flag != NULL; flag++)
931 xfree((
void *)cc_added_args); cc_added_args = NULL;
932 xfree(tmp_dir); tmp_dir = NULL;
933 xfree(pch_file); pch_file = NULL;
936 free_list(&unit_queue, close_handle_p);
937 free_list(&active_units, close_handle_p);
938 free_list(&compact_units, close_handle_p);
939 free_list(&stale_units, close_handle_p);
942 mjit_enabled =
false;
943 verbose(1,
"Successful MJIT finish");
958 RUBY_MARK_ENTER(
"mjit");
965 CRITICAL_SECTION_START(4,
"mjit_mark");
967 if (compiling_iseqs != NULL) {
968 while (compiling_iseqs[length]) length++;
970 length += active_units.length;
975 if (compiling_iseqs != NULL) {
976 while (compiling_iseqs[i]) {
977 iseqs[i] = compiling_iseqs[i];
981 list_for_each(&active_units.head, unit, unode) {
982 iseqs[i] = unit->iseq;
986 CRITICAL_SECTION_FINISH(4,
"mjit_mark");
988 for (i = 0; i < length; i++) {
989 if (iseqs[i] == NULL)
994 RUBY_MARK_LEAVE(
"mjit");
1002 if (body->jit_unit && (cc_entries = body->jit_unit->cc_entries) != NULL) {
1004 for (
unsigned int i = 0; i < body->jit_unit->cc_entries_size; i++) {
1006 if (cc != NULL && vm_cc_markable(cc)) {
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
#define xfree
Old name of ruby_xfree.
#define Qundef
Old name of RUBY_Qundef.
#define xmalloc
Old name of ruby_xmalloc.
#define FIX2INT
Old name of RB_FIX2INT.
#define Qtrue
Old name of RUBY_Qtrue.
#define Qfalse
Old name of RUBY_Qfalse.
void rb_raise(VALUE exc, const char *fmt,...)
Exception entry point.
void rb_gc_mark(VALUE obj)
Marks an object.
void rb_memerror(void)
Triggers out-of-memory error.
VALUE rb_gc_location(VALUE obj)
Finds a new "location" of an object.
int rb_path_check(const char *path)
This function is mysterious.
int rb_cloexec_open(const char *pathname, int flags, mode_t mode)
Opens a file that closes on exec.
void rb_thread_wait_for(struct timeval time)
Identical to rb_thread_sleep(), except it takes struct timeval instead.
char * ruby_strdup(const char *str)
This is our own version of strdup(3) that uses ruby_xmalloc() instead of system malloc (benefits our ...
#define strdup(s)
Just another name of ruby_strdup.
#define ALLOCA_N(type, n)
#define StringValuePtr(v)
Identical to StringValue, except it returns a char*.
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.