diff --git a/core/unix/os.c b/core/unix/os.c index ffb1f5649d6..715461a8d3d 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -9371,7 +9371,6 @@ post_system_call(dcontext_t *dcontext) /* Lazy rseq handling. */ if (success) { rseq_process_syscall(dcontext); - rseq_locate_rseq_regions(); } break; #endif @@ -10732,7 +10731,7 @@ os_take_over_all_unknown_threads(dcontext_t *dcontext) #ifdef LINUX /* Check this thread for rseq in between setup and start. */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); #endif /* Find tids for which we have no thread record, meaning they are not under @@ -10960,7 +10959,7 @@ os_thread_take_over(priv_mcontext_t *mc, kernel_sigset_t *sigset) * regions as rseq when the rseq syscall is never set up. */ if (rseq_is_registered_for_current_thread()) { - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); rseq_thread_attach(dcontext); } #endif diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index c0c691644e9..3537a5396bb 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -51,6 +51,8 @@ #include "include/syscall.h" #include +#define ABS(x) ((x) < 0 ? -(x) : (x)) + /* The linux/rseq.h header made a source-breaking change in torvalds/linux@bfdf4e6 * which broke our build. To avoid future issues we use our own definitions. * Binary breakage is unlikely without long periods of deprecation so this is @@ -78,6 +80,26 @@ DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, INIT_LOCK_FREE(rseq_trigger_lock)); static volatile bool rseq_enabled; +/* The struct rseq registered by glibc is present in the struct pthread. + * As of glibc 2.35, it is present at the following offset from the app + * lib seg base. We check these offsets first and then fall back to a + * wider search. The linux.rseq test helps detect changes in these + * offsets in future glibc versions. + */ +#ifdef X86 +# ifdef X64 +# define GLIBC_RSEQ_OFFSET 2464 +# else +# define GLIBC_RSEQ_OFFSET 1312 +# endif +#else +/* This was verified on AArch64, but not on AArch32. + * XXX: To improve struct rseq offset detection on AArch32, find the offset + * on an AArch32 machine running glibc 2.35+ and add here. + */ +# define GLIBC_RSEQ_OFFSET -32 +#endif + /* We require all threads to use the same TLS offset to point at struct rseq. */ static int rseq_tls_offset; @@ -163,7 +185,7 @@ d_r_rseq_init(void) rseq_cs_free _IF_DEBUG("rseq_cs table")); /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); } void @@ -229,7 +251,7 @@ static void rseq_clear_tls_ptr(dcontext_t *dcontext) { ASSERT(rseq_tls_offset != 0); - byte *base = get_segment_base(LIB_SEG_TLS); + byte *base = get_app_segment_base(LIB_SEG_TLS); struct rseq *app_rseq = (struct rseq *)(base + rseq_tls_offset); /* We're directly writing this in the cache, so we do not bother with safe_read * or safe_write here either. We already cannot handle rseq adversarial cases. @@ -545,7 +567,7 @@ rseq_process_elf_sections(module_area_t *ma, bool at_map, /* Returns whether successfully searched for rseq data (not whether found rseq data). */ static bool -rseq_process_module(module_area_t *ma, bool at_map) +rseq_process_module(module_area_t *ma, bool at_map, bool saw_glibc_rseq_reg) { bool res = false; ASSERT(is_elf_so_header(ma->start, ma->end - ma->start)); @@ -596,7 +618,11 @@ rseq_process_module(module_area_t *ma, bool at_map) goto rseq_process_module_cleanup; strtab = (char *)(str_map + sec_hdr[elf_hdr->e_shstrndx].sh_offset - offs); } - rseq_process_elf_sections(ma, at_map, sec_hdr, strtab, load_offs); + /* When saw_glibc_rseq_reg is set, we are still at glibc init, and ld has not + * relocated the executable yet. + */ + rseq_process_elf_sections(ma, at_map || saw_glibc_rseq_reg, sec_hdr, strtab, + load_offs); res = true; rseq_process_module_cleanup: if (str_size != 0) @@ -621,6 +647,31 @@ rseq_process_module(module_area_t *ma, bool at_map) return res; } +static bool +try_struct_rseq(void *try_addr) +{ + static const int RSEQ_RARE_SIGNATURE = 42; + int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), + RSEQ_FLAG_UNREGISTER, RSEQ_RARE_SIGNATURE); + LOG(GLOBAL, LOG_LOADER, 3, "Tried rseq @ " PFX " => %d\n", try_addr, res); + if (res == -EINVAL) /* Our struct != registered struct. */ + return false; + /* We expect -EPERM on a signature mismatch. On the small chance the app + * actually used 42 for its signature we'll have to re-register it. + */ + if (res == 0) { + int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), 0, + RSEQ_RARE_SIGNATURE); + ASSERT(res == 0); + res = -EPERM; + } + if (res == -EPERM) { + /* Found it! */ + return true; + } + return false; +} + /* If we did not observe the app invoke SYS_rseq (because we attached mid-run) * we must search for its TLS location. */ @@ -638,6 +689,19 @@ rseq_locate_tls_offset(void) */ int offset = 0; byte *addr = get_app_segment_base(LIB_SEG_TLS); + if (addr > 0) { + byte *try_glibc_addr = addr + GLIBC_RSEQ_OFFSET; + if (try_struct_rseq(try_glibc_addr)) { + LOG(GLOBAL, LOG_LOADER, 2, + "Found glibc struct rseq @ " PFX " for thread => %s:%s0x%x\n", + try_glibc_addr, get_register_name(LIB_SEG_TLS), + (GLIBC_RSEQ_OFFSET < 0 ? "-" : ""), ABS(GLIBC_RSEQ_OFFSET)); + return GLIBC_RSEQ_OFFSET; + } + } + /* Either the app's glibc does not have rseq support (old glibc or disabled by app) + * or the offset of glibc's struct rseq has changed. We do a wider search now. + */ byte *seg_bottom; size_t seg_size; if (addr > 0 && get_memory_info(addr, &seg_bottom, &seg_size, NULL)) { @@ -646,41 +710,31 @@ rseq_locate_tls_offset(void) /* struct rseq_cs is aligned to 32. */ int alignment = __alignof(struct rseq_cs); int i; - /* For x86, static TLS is at a negative offset from the app library segment - * base, while for aarchxx it is positive. + /* When rseq support is enabled in glibc 2.35+, the glibc-registered struct rseq + * is present in the struct pthread, which is at a positive offset from the + * app library segment base on x86, and negative on aarchxx. However, in the + * absence of rseq support from glibc, the app manually registers its own + * struct rseq which is present in static TLS, which is at a negative offset + * from the app library segment base on x86, and positive on aarchxx. */ - for (i = 0; IF_X86_ELSE(addr - i * alignment >= seg_bottom, - addr + i * alignment < seg_bottom + seg_size); - i++) { - byte *try_addr = IF_X86_ELSE(addr - i * alignment, addr + i * alignment); - ASSERT(try_addr >= seg_bottom); /* For loop guarantees this. */ - /* Our strategy is to check all of the aligned static TLS addresses to - * find the registered one. Our caller is not supposed to call here - * until the app has registered the current thread. + ASSERT(seg_bottom <= addr && addr < seg_bottom + seg_size); + for (i = (seg_bottom - addr) / alignment; + addr + i * alignment < seg_bottom + seg_size; ++i) { + byte *try_addr = addr + i * alignment; + ASSERT(seg_bottom <= try_addr && + try_addr < seg_bottom + seg_size); /* For loop guarantees this. */ + /* Our strategy is to check all of the aligned addresses to find the + * registered one. Our caller is not supposed to call here until the app + * has registered the current thread (either manually or using glibc). */ - static const int RSEQ_RARE_SIGNATURE = 42; - int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), - RSEQ_FLAG_UNREGISTER, RSEQ_RARE_SIGNATURE); - LOG(GLOBAL, LOG_LOADER, 3, "Tried rseq @ " PFX " => %d\n", try_addr, res); - if (res == -EINVAL) /* Our struct != registered struct. */ - continue; - /* We expect -EPERM on a signature mismatch. On the small chance the app - * actually used 42 for its signature we'll have to re-register it. - */ - if (res == 0) { - int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), 0, - RSEQ_RARE_SIGNATURE); - ASSERT(res == 0); - res = -EPERM; - } - if (res == -EPERM) { - /* Found it! */ + if (try_struct_rseq(try_addr)) { LOG(GLOBAL, LOG_LOADER, 2, "Found struct rseq @ " PFX " for thread => %s:%s0x%x\n", try_addr, - get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), i * alignment); - offset = IF_X86_ELSE(-i * alignment, i * alignment); + get_register_name(LIB_SEG_TLS), (i < 0 ? "-" : ""), + ABS(i) * alignment); + offset = i * alignment; + break; } - break; } } return offset; @@ -692,25 +746,34 @@ rseq_process_syscall(dcontext_t *dcontext) byte *seg_base = get_app_segment_base(LIB_SEG_TLS); byte *app_addr = (byte *)dcontext->sys_param0; bool constant_offset = false; + bool first_rseq_registration = false; if (rseq_tls_offset == 0) { SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); int offset = app_addr - seg_base; /* To handle races here, we use an atomic_exchange. */ int prior = atomic_exchange_int(&rseq_tls_offset, offset); SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); + if (prior == 0) + first_rseq_registration = true; constant_offset = (prior == 0 || prior == offset); LOG(GLOBAL, LOG_LOADER, 2, - "Observed struct rseq @ " PFX " for thread => %s:%s0x%x\n", app_addr, - get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), - IF_X86_ELSE(-rseq_tls_offset, rseq_tls_offset)); + "Observed struct rseq at syscall @ " PFX " for thread => %s:%s0x%x\n", + app_addr, get_register_name(LIB_SEG_TLS), (rseq_tls_offset < 0 ? "-" : ""), + ABS(rseq_tls_offset)); } else constant_offset = (seg_base + rseq_tls_offset == app_addr); if (!constant_offset) { - REPORT_FATAL_ERROR_AND_EXIT( - RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), get_application_pid(), - "struct rseq is not always in static thread-local storage"); + REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), + get_application_pid(), + "struct rseq is not always at the same offset"); ASSERT_NOT_REACHED(); } + /* The struct rseq registered by glibc 2.35+ is inside struct pthread, which is + * at a positive offset from thread pointer on x86 and negative offset on AArch64, + * both unlike the static TLS used by manual app registration. + */ + rseq_locate_rseq_regions(first_rseq_registration && + IF_X86_ELSE(rseq_tls_offset > 0, rseq_tls_offset < 0)); } /* Restartable sequence region identification. @@ -723,7 +786,7 @@ rseq_process_syscall(dcontext_t *dcontext) * the app has registered the current thread for rseq. */ void -rseq_locate_rseq_regions(void) +rseq_locate_rseq_regions(bool saw_glibc_rseq_reg) { if (rseq_enabled) return; @@ -759,7 +822,7 @@ rseq_locate_rseq_regions(void) module_iterator_t *iter = module_iterator_start(); while (module_iterator_hasnext(iter)) { module_area_t *ma = module_iterator_next(iter); - rseq_process_module(ma, false /*!at_map*/); + rseq_process_module(ma, false /*!at_map*/, saw_glibc_rseq_reg); } module_iterator_stop(iter); d_r_mutex_unlock(&rseq_trigger_lock); @@ -769,7 +832,7 @@ void rseq_module_init(module_area_t *ma, bool at_map) { if (rseq_enabled) { - rseq_process_module(ma, at_map); + rseq_process_module(ma, at_map, false); } } diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index 56a25aa4dd9..e28bfb7b9c1 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -58,7 +58,7 @@ bool rseq_is_registered_for_current_thread(void); void -rseq_locate_rseq_regions(void); +rseq_locate_rseq_regions(bool saw_glibc_rseq_reg); void rseq_module_init(module_area_t *ma, bool at_map); diff --git a/suite/tests/CMakeLists.txt b/suite/tests/CMakeLists.txt index c4a52ba9308..1ea3fe5a0d4 100644 --- a/suite/tests/CMakeLists.txt +++ b/suite/tests/CMakeLists.txt @@ -5403,3 +5403,13 @@ if (BUILD_CLIENTS AND NOT ANDROID) code_api|tool.drcacheoff.simple PROPERTIES LABELS UBUNTU_22) endif () +if (LINUX AND X64 AND HAVE_RSEQ) + set_tests_properties( + code_api|api.rseq + code_api|linux.rseq_disable + code_api|linux.rseq_noarray + code_api|linux.rseq + code_api|linux.rseq_table + code_api|tool.drcacheoff.rseq + PROPERTIES LABELS UBUNTU_22) +endif () diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index 81cf72a9c93..e6e5650895b 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -58,6 +58,12 @@ # define _GNU_SOURCE #endif #include +#if defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 35)) +# include +# define GLIBC_RSEQ 1 +#else +# define RSEQ_SIG IF_X86_ELSE(0x90909090, 0xd503201f) /* nops to disasm nicely */ +#endif #include #include #undef NDEBUG @@ -66,7 +72,6 @@ #define EXPANDSTR(x) #x #define STRINGIFY(x) EXPANDSTR(x) -#define RSEQ_SIG IF_X86_ELSE(0x90909090, 0xd503201f) /* nops to disasm nicely */ #ifdef RSEQ_TEST_USE_OLD_SECTION_NAME # define RSEQ_SECTION_NAME "__rseq_table" #else @@ -107,6 +112,56 @@ static void *thread_ready; static volatile int sigill_count; +static bool +register_rseq() +{ +#ifdef GLIBC_RSEQ + if (__rseq_size > 0) { + /* Our glibc rseq handling in rseq_linux.c checks the following offset + * first for the glibc-registered struct rseq. Though we do have a + * fallback that does a wider search, it would be good to keep the + * expected offset in sync with glibc changes. + */ +# ifdef X86 +# ifdef X64 + assert(__rseq_offset == 2464); +# else + assert(__rseq_offset == 1312); +# endif +# else + assert(__rseq_offset == -32); +# endif + /* Already registered by glibc. Verify that it's there. */ + struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; + /* Glibc uses RSEQ_SIG as the signature. The following will return + * EBUSY if reg_rseq is the registered struct rseq and EINVAL otherwise. + * FTR: if we use some other value as signature (like zero), this + * becomes an EPERM vs EINVAL check. + */ + int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, RSEQ_SIG); + assert(res == -1 && (errno == EBUSY || errno == ENOSYS)); + } else { +#endif + rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; + int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); + assert(res == 0 || (res == -1 && errno == ENOSYS)); +#ifdef GLIBC_RSEQ + } +#endif + /* Linux kernel 4.18+ is required. */ + return errno != ENOSYS; +} + +static volatile struct rseq * +get_my_rseq() +{ +#ifdef GLIBC_RSEQ + if (__rseq_size > 0) + return (struct rseq *)(__builtin_thread_pointer() + __rseq_offset); +#endif + return &rseq_tls; +} + static void signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt) { @@ -117,6 +172,7 @@ signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt) static void test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_out) { + volatile struct rseq *reg_rseq = get_my_rseq(); /* We use static to avoid stack reference issues with our extra frame inside the asm. */ static __u32 id = RSEQ_CPU_ID_UNINITIALIZED; @@ -141,7 +197,7 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_simple(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test register inputs to the sequence. */ "movl %[cpu_id], %%eax\n\t" "movq %%rsp, %%rcx\n\t" @@ -193,12 +249,12 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), + : [cpu_id] "m"(reg_rseq->cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), [force_restart] "m"(force_restart) : "rax", "rcx", "rdx", "rbx", "rdi", "rsi", "memory"); #elif defined(AARCH64) @@ -218,7 +274,7 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o * inputs but that's simpler than manually computing all these * addresses inside the sequence. */ - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -262,12 +318,12 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), + : [cpu_id] "m"(reg_rseq->cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), [force_restart] "m"(force_restart) : "x0", "x1", "memory"); #else @@ -294,6 +350,7 @@ test_rseq_call(void) static void test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_out) { + volatile struct rseq *reg_rseq = get_my_rseq(); __u32 id = RSEQ_CPU_ID_UNINITIALIZED; int completions = 0; int restarts = 0; @@ -306,7 +363,7 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_branches(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "movl %[cpu_id], %%eax\n\t" /* Test "falling into" the rseq region. */ @@ -354,13 +411,13 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "13:\n\t" "12:\n\t" "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [force_restart] "m"(force_restart) + : [cpu_id] "m"(reg_rseq->cpu_id), [force_restart] "m"(force_restart) : "rax", "rcx", "memory"); #elif defined(AARCH64) __asm__ __volatile__( @@ -372,7 +429,7 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ /* Store the entry into the ptr. */ "adrp x0, rseq_cs_branches\n\t" "add x0, x0, :lo12:rseq_cs_branches\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -424,18 +481,20 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "13:\n\t" "12:\n\t" "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [force_restart] "m"(force_restart) + : [cpu_id] "m"(reg_rseq->cpu_id), [force_restart] "m"(force_restart) : "x0", "x1", "memory"); #else # error Unsupported arch #endif assert(id != RSEQ_CPU_ID_UNINITIALIZED); + *completions_out = completions; + *restarts_out = restarts; } static void @@ -461,6 +520,7 @@ test_rseq_branches(void) static void test_rseq_native_fault(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); int restarts = 0; #ifdef X86 __asm__ __volatile__( @@ -471,7 +531,7 @@ test_rseq_native_fault(void) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_fault(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" "pxor %%xmm0, %%xmm0\n\t" "mov $1,%%rcx\n\t" "movq %%rcx, %%xmm1\n\t" @@ -507,10 +567,10 @@ test_rseq_native_fault(void) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : : "rax", "rcx", "xmm0", "xmm1", "memory"); #elif defined(AARCH64) @@ -523,7 +583,7 @@ test_rseq_native_fault(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_fault\n\t" "add x0, x0, :lo12:rseq_cs_fault\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" "eor v0.16b, v0.16b, v0.16b\n\t" "mov x1, #1\n\t" "mov v1.D[0], x1\n\t" @@ -561,10 +621,10 @@ test_rseq_native_fault(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : : "x0", "x1", "q0", "q1", "memory"); #else @@ -585,6 +645,7 @@ test_rseq_native_fault(void) static void test_rseq_native_abort(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); #ifdef DEBUG /* See above: special code in core/ is DEBUG-only> */ int restarts = 0; # ifdef X86 @@ -596,7 +657,7 @@ test_rseq_native_abort(void) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_abort(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" "pxor %%xmm0, %%xmm0\n\t" "mov $1,%%rcx\n\t" "movq %%rcx, %%xmm1\n\t" @@ -647,13 +708,13 @@ test_rseq_native_abort(void) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : [cpu_mask_size] "i"(sizeof(cpu_set_t)), [sysnum_setaffinity] "i"(SYS_sched_setaffinity) - : "rax", "rcx", "rdx", "xmm0", "xmm1", "memory"); + : "rax", "rcx", "rdx", "rsi", "rdi", "xmm0", "xmm1", "memory"); # elif defined(AARCH64) __asm__ __volatile__( /* clang-format off */ /* (avoid indenting next few lines) */ @@ -664,7 +725,7 @@ test_rseq_native_abort(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_abort\n\t" "add x0, x0, :lo12:rseq_cs_abort\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" "eor v0.16b, v0.16b, v0.16b\n\t" "mov x1, #1\n\t" "mov v1.D[0], x1\n\t" @@ -720,10 +781,10 @@ test_rseq_native_abort(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : [cpu_mask_size] "i"(sizeof(cpu_set_t)), [sysnum_setaffinity] "i"(SYS_sched_setaffinity) : "x0", "x1", "x2", "x8", "q0", "q1", "memory"); @@ -739,6 +800,7 @@ test_rseq_native_abort(void) static void test_rseq_writeback_store(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); __u32 id = RSEQ_CPU_ID_UNINITIALIZED; int array[2] = { 0 }; int *index = array; @@ -751,7 +813,7 @@ test_rseq_writeback_store(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_writeback\n\t" "add x0, x0, :lo12:rseq_cs_writeback\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -784,11 +846,11 @@ test_rseq_writeback_store(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), [index] "=g"(index) - : [cpu_id] "m"(rseq_tls.cpu_id) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [index] "=g"(index) + : [cpu_id] "m"(reg_rseq->cpu_id) : "x0", "x1", "x2", "x28", "memory"); assert(id != RSEQ_CPU_ID_UNINITIALIZED); } @@ -802,10 +864,9 @@ rseq_thread_loop(void *arg) * in this function is close enough: the test already has non-determinism. */ signal_cond_var(thread_ready); - rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; - int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); - if (res != 0) - return NULL; + bool res = register_rseq(); + assert(res); + volatile struct rseq *reg_rseq = get_my_rseq(); static int zero; # ifdef X86 __asm__ __volatile__( @@ -816,7 +877,7 @@ rseq_thread_loop(void *arg) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_thread(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test "falling into" the rseq region. */ /* Restartable sequence. We loop to ensure we're in the region on @@ -851,10 +912,10 @@ rseq_thread_loop(void *arg) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [zero] "=m"(zero) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [zero] "=m"(zero) : [exit_requested] "m"(exit_requested) : "rax", "memory"); # elif defined(AARCH64) @@ -867,7 +928,7 @@ rseq_thread_loop(void *arg) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_thread\n\t" "add x0, x0, :lo12:rseq_cs_thread\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test "falling into" the rseq region. */ /* Restartable sequence. We loop to ensure we're in the region on @@ -902,10 +963,10 @@ rseq_thread_loop(void *arg) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [zero] "=m"(zero) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [zero] "=m"(zero) : [exit_requested] "m"(exit_requested) : "x0", "x1", "memory"); # else @@ -963,7 +1024,7 @@ kernel_xfer_event(void *drcontext, const dr_kernel_xfer_info_t *info) DR_EXPORT void dr_client_main(client_id_t id, int argc, const char *argv[]) { - /* Ensure DR_XFER_RSEQ_ABORT is rasied. */ + /* Ensure DR_XFER_RSEQ_ABORT is raised. */ dr_register_kernel_xfer_event(kernel_xfer_event); drmemtrace_client_main(id, argc, argv); } @@ -973,9 +1034,7 @@ int main() { intercept_signal(SIGILL, signal_handler, false); - rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; - int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); - if (res == 0) { + if (register_rseq()) { #ifdef RSEQ_TEST_ATTACH /* Set -offline to avoid trying to open a pipe to a missing reader. */ if (setenv("DYNAMORIO_OPTIONS", "-stderr_mask 0xc -client_lib ';;-offline'", @@ -1011,9 +1070,6 @@ main() join_thread(mythread); destroy_cond_var(thread_ready); #endif - } else { - /* Linux kernel 4.18+ is required. */ - assert(errno == ENOSYS); } print("All done\n"); return 0; diff --git a/tools/drdeploy.c b/tools/drdeploy.c index 8f751931895..6d82b8a5ddd 100644 --- a/tools/drdeploy.c +++ b/tools/drdeploy.c @@ -1282,20 +1282,6 @@ _tmain(int argc, TCHAR *targv[]) /* we re-read the tool list if the root, platform or toolconfig dir change */ read_tool_list(dr_toolconfig_dir, dr_platform); -#if defined(LINUX) && !defined(ANDROID) && (defined(DRCONFIG) || defined(DRRUN)) - /* XXX i#5431: Workaround for an rseq issue with glibc 2.35 which makes many - * apps fail up front. We expect to remove this once the real fix is in place. - */ -# include - const char *libc_ver = gnu_get_libc_version(); - if (libc_ver[0] != '\0' && libc_ver[0] >= '2' && libc_ver[1] == '.' && - libc_ver[2] >= '3' && libc_ver[3] >= '5') { - info("glibc2.35 detected: setting -disable_rseq"); - add_extra_option(extra_ops, BUFFER_SIZE_ELEMENTS(extra_ops), &extra_ops_sofar, - "-disable_rseq"); - } -#endif - /* parse command line */ for (i = 1; i < argc; i++) {