From 5ef7b9e4bf77fe28e8353238e5a397b84f38199e Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 18 Nov 2022 00:04:04 -0500 Subject: [PATCH] i#5431: Support glibc's rseq support (#5711) Fixes issues with DR's rseq handling in glibc 2.35+. Glibc 2.35 added support for the Linux rseq feature. See https://lwn.net/Articles/883104/ for details. TLDR; glibc registers its own struct rseq at init time, and stores its offset from the thread pointer in __rseq_offset. The glibc-registered struct rseq is present in the struct pthread. If glibc's rseq support isn't available, either due to some issue or because the user disabled it by exporting GLIBC_TUNABLES=glibc.pthread.rseq=0, it will set __rseq_size to zero. Improves the heuristic to find the registered struct rseq. For the glibc-support case: on AArch64, it is at a -ve offset from app lib seg base, whereas on x86 it's at a +ve offset. On both AArch64 and x86, the offset is of the opposite sign than what it would be if the app registered the struct rseq manually in its static TLS (which happens for older glibc and when glibc's rseq support is disabled). Detects whether the glibc rseq support is enabled by looking at the sign of the struct rseq offset. Removes the drrun -disable_rseq workaround added by #5695. Adjusts the linux.rseq test to get the struct rseq registered by glibc, when it's available. Also fixes some issues in the test. Adds the Ubuntu_22 tag to rseq tests so that they are enabled. Our Ubuntu-20 CI tests the case without rseq support in glibc, where the app registers the struct rseq. This also helps test the case where the app is not using glibc. Also, our Ubuntu-22 CI tests the case with Glibc rseq support. Manually tested the disabled rseq support case on glibc 2.35, but not adding a CI version of it. Fixes #5431 --- core/unix/os.c | 5 +- core/unix/rseq_linux.c | 149 ++++++++++++++++++++++++---------- core/unix/rseq_linux.h | 2 +- suite/tests/CMakeLists.txt | 10 +++ suite/tests/linux/rseq.c | 158 +++++++++++++++++++++++++------------ tools/drdeploy.c | 14 ---- 6 files changed, 226 insertions(+), 112 deletions(-) diff --git a/core/unix/os.c b/core/unix/os.c index ffb1f5649d6..715461a8d3d 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -9371,7 +9371,6 @@ post_system_call(dcontext_t *dcontext) /* Lazy rseq handling. */ if (success) { rseq_process_syscall(dcontext); - rseq_locate_rseq_regions(); } break; #endif @@ -10732,7 +10731,7 @@ os_take_over_all_unknown_threads(dcontext_t *dcontext) #ifdef LINUX /* Check this thread for rseq in between setup and start. */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); #endif /* Find tids for which we have no thread record, meaning they are not under @@ -10960,7 +10959,7 @@ os_thread_take_over(priv_mcontext_t *mc, kernel_sigset_t *sigset) * regions as rseq when the rseq syscall is never set up. */ if (rseq_is_registered_for_current_thread()) { - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); rseq_thread_attach(dcontext); } #endif diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index c0c691644e9..3537a5396bb 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -51,6 +51,8 @@ #include "include/syscall.h" #include +#define ABS(x) ((x) < 0 ? -(x) : (x)) + /* The linux/rseq.h header made a source-breaking change in torvalds/linux@bfdf4e6 * which broke our build. To avoid future issues we use our own definitions. * Binary breakage is unlikely without long periods of deprecation so this is @@ -78,6 +80,26 @@ DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, INIT_LOCK_FREE(rseq_trigger_lock)); static volatile bool rseq_enabled; +/* The struct rseq registered by glibc is present in the struct pthread. + * As of glibc 2.35, it is present at the following offset from the app + * lib seg base. We check these offsets first and then fall back to a + * wider search. The linux.rseq test helps detect changes in these + * offsets in future glibc versions. + */ +#ifdef X86 +# ifdef X64 +# define GLIBC_RSEQ_OFFSET 2464 +# else +# define GLIBC_RSEQ_OFFSET 1312 +# endif +#else +/* This was verified on AArch64, but not on AArch32. + * XXX: To improve struct rseq offset detection on AArch32, find the offset + * on an AArch32 machine running glibc 2.35+ and add here. + */ +# define GLIBC_RSEQ_OFFSET -32 +#endif + /* We require all threads to use the same TLS offset to point at struct rseq. */ static int rseq_tls_offset; @@ -163,7 +185,7 @@ d_r_rseq_init(void) rseq_cs_free _IF_DEBUG("rseq_cs table")); /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); } void @@ -229,7 +251,7 @@ static void rseq_clear_tls_ptr(dcontext_t *dcontext) { ASSERT(rseq_tls_offset != 0); - byte *base = get_segment_base(LIB_SEG_TLS); + byte *base = get_app_segment_base(LIB_SEG_TLS); struct rseq *app_rseq = (struct rseq *)(base + rseq_tls_offset); /* We're directly writing this in the cache, so we do not bother with safe_read * or safe_write here either. We already cannot handle rseq adversarial cases. @@ -545,7 +567,7 @@ rseq_process_elf_sections(module_area_t *ma, bool at_map, /* Returns whether successfully searched for rseq data (not whether found rseq data). */ static bool -rseq_process_module(module_area_t *ma, bool at_map) +rseq_process_module(module_area_t *ma, bool at_map, bool saw_glibc_rseq_reg) { bool res = false; ASSERT(is_elf_so_header(ma->start, ma->end - ma->start)); @@ -596,7 +618,11 @@ rseq_process_module(module_area_t *ma, bool at_map) goto rseq_process_module_cleanup; strtab = (char *)(str_map + sec_hdr[elf_hdr->e_shstrndx].sh_offset - offs); } - rseq_process_elf_sections(ma, at_map, sec_hdr, strtab, load_offs); + /* When saw_glibc_rseq_reg is set, we are still at glibc init, and ld has not + * relocated the executable yet. + */ + rseq_process_elf_sections(ma, at_map || saw_glibc_rseq_reg, sec_hdr, strtab, + load_offs); res = true; rseq_process_module_cleanup: if (str_size != 0) @@ -621,6 +647,31 @@ rseq_process_module(module_area_t *ma, bool at_map) return res; } +static bool +try_struct_rseq(void *try_addr) +{ + static const int RSEQ_RARE_SIGNATURE = 42; + int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), + RSEQ_FLAG_UNREGISTER, RSEQ_RARE_SIGNATURE); + LOG(GLOBAL, LOG_LOADER, 3, "Tried rseq @ " PFX " => %d\n", try_addr, res); + if (res == -EINVAL) /* Our struct != registered struct. */ + return false; + /* We expect -EPERM on a signature mismatch. On the small chance the app + * actually used 42 for its signature we'll have to re-register it. + */ + if (res == 0) { + int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), 0, + RSEQ_RARE_SIGNATURE); + ASSERT(res == 0); + res = -EPERM; + } + if (res == -EPERM) { + /* Found it! */ + return true; + } + return false; +} + /* If we did not observe the app invoke SYS_rseq (because we attached mid-run) * we must search for its TLS location. */ @@ -638,6 +689,19 @@ rseq_locate_tls_offset(void) */ int offset = 0; byte *addr = get_app_segment_base(LIB_SEG_TLS); + if (addr > 0) { + byte *try_glibc_addr = addr + GLIBC_RSEQ_OFFSET; + if (try_struct_rseq(try_glibc_addr)) { + LOG(GLOBAL, LOG_LOADER, 2, + "Found glibc struct rseq @ " PFX " for thread => %s:%s0x%x\n", + try_glibc_addr, get_register_name(LIB_SEG_TLS), + (GLIBC_RSEQ_OFFSET < 0 ? "-" : ""), ABS(GLIBC_RSEQ_OFFSET)); + return GLIBC_RSEQ_OFFSET; + } + } + /* Either the app's glibc does not have rseq support (old glibc or disabled by app) + * or the offset of glibc's struct rseq has changed. We do a wider search now. + */ byte *seg_bottom; size_t seg_size; if (addr > 0 && get_memory_info(addr, &seg_bottom, &seg_size, NULL)) { @@ -646,41 +710,31 @@ rseq_locate_tls_offset(void) /* struct rseq_cs is aligned to 32. */ int alignment = __alignof(struct rseq_cs); int i; - /* For x86, static TLS is at a negative offset from the app library segment - * base, while for aarchxx it is positive. + /* When rseq support is enabled in glibc 2.35+, the glibc-registered struct rseq + * is present in the struct pthread, which is at a positive offset from the + * app library segment base on x86, and negative on aarchxx. However, in the + * absence of rseq support from glibc, the app manually registers its own + * struct rseq which is present in static TLS, which is at a negative offset + * from the app library segment base on x86, and positive on aarchxx. */ - for (i = 0; IF_X86_ELSE(addr - i * alignment >= seg_bottom, - addr + i * alignment < seg_bottom + seg_size); - i++) { - byte *try_addr = IF_X86_ELSE(addr - i * alignment, addr + i * alignment); - ASSERT(try_addr >= seg_bottom); /* For loop guarantees this. */ - /* Our strategy is to check all of the aligned static TLS addresses to - * find the registered one. Our caller is not supposed to call here - * until the app has registered the current thread. + ASSERT(seg_bottom <= addr && addr < seg_bottom + seg_size); + for (i = (seg_bottom - addr) / alignment; + addr + i * alignment < seg_bottom + seg_size; ++i) { + byte *try_addr = addr + i * alignment; + ASSERT(seg_bottom <= try_addr && + try_addr < seg_bottom + seg_size); /* For loop guarantees this. */ + /* Our strategy is to check all of the aligned addresses to find the + * registered one. Our caller is not supposed to call here until the app + * has registered the current thread (either manually or using glibc). */ - static const int RSEQ_RARE_SIGNATURE = 42; - int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), - RSEQ_FLAG_UNREGISTER, RSEQ_RARE_SIGNATURE); - LOG(GLOBAL, LOG_LOADER, 3, "Tried rseq @ " PFX " => %d\n", try_addr, res); - if (res == -EINVAL) /* Our struct != registered struct. */ - continue; - /* We expect -EPERM on a signature mismatch. On the small chance the app - * actually used 42 for its signature we'll have to re-register it. - */ - if (res == 0) { - int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), 0, - RSEQ_RARE_SIGNATURE); - ASSERT(res == 0); - res = -EPERM; - } - if (res == -EPERM) { - /* Found it! */ + if (try_struct_rseq(try_addr)) { LOG(GLOBAL, LOG_LOADER, 2, "Found struct rseq @ " PFX " for thread => %s:%s0x%x\n", try_addr, - get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), i * alignment); - offset = IF_X86_ELSE(-i * alignment, i * alignment); + get_register_name(LIB_SEG_TLS), (i < 0 ? "-" : ""), + ABS(i) * alignment); + offset = i * alignment; + break; } - break; } } return offset; @@ -692,25 +746,34 @@ rseq_process_syscall(dcontext_t *dcontext) byte *seg_base = get_app_segment_base(LIB_SEG_TLS); byte *app_addr = (byte *)dcontext->sys_param0; bool constant_offset = false; + bool first_rseq_registration = false; if (rseq_tls_offset == 0) { SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); int offset = app_addr - seg_base; /* To handle races here, we use an atomic_exchange. */ int prior = atomic_exchange_int(&rseq_tls_offset, offset); SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); + if (prior == 0) + first_rseq_registration = true; constant_offset = (prior == 0 || prior == offset); LOG(GLOBAL, LOG_LOADER, 2, - "Observed struct rseq @ " PFX " for thread => %s:%s0x%x\n", app_addr, - get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), - IF_X86_ELSE(-rseq_tls_offset, rseq_tls_offset)); + "Observed struct rseq at syscall @ " PFX " for thread => %s:%s0x%x\n", + app_addr, get_register_name(LIB_SEG_TLS), (rseq_tls_offset < 0 ? "-" : ""), + ABS(rseq_tls_offset)); } else constant_offset = (seg_base + rseq_tls_offset == app_addr); if (!constant_offset) { - REPORT_FATAL_ERROR_AND_EXIT( - RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), get_application_pid(), - "struct rseq is not always in static thread-local storage"); + REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), + get_application_pid(), + "struct rseq is not always at the same offset"); ASSERT_NOT_REACHED(); } + /* The struct rseq registered by glibc 2.35+ is inside struct pthread, which is + * at a positive offset from thread pointer on x86 and negative offset on AArch64, + * both unlike the static TLS used by manual app registration. + */ + rseq_locate_rseq_regions(first_rseq_registration && + IF_X86_ELSE(rseq_tls_offset > 0, rseq_tls_offset < 0)); } /* Restartable sequence region identification. @@ -723,7 +786,7 @@ rseq_process_syscall(dcontext_t *dcontext) * the app has registered the current thread for rseq. */ void -rseq_locate_rseq_regions(void) +rseq_locate_rseq_regions(bool saw_glibc_rseq_reg) { if (rseq_enabled) return; @@ -759,7 +822,7 @@ rseq_locate_rseq_regions(void) module_iterator_t *iter = module_iterator_start(); while (module_iterator_hasnext(iter)) { module_area_t *ma = module_iterator_next(iter); - rseq_process_module(ma, false /*!at_map*/); + rseq_process_module(ma, false /*!at_map*/, saw_glibc_rseq_reg); } module_iterator_stop(iter); d_r_mutex_unlock(&rseq_trigger_lock); @@ -769,7 +832,7 @@ void rseq_module_init(module_area_t *ma, bool at_map) { if (rseq_enabled) { - rseq_process_module(ma, at_map); + rseq_process_module(ma, at_map, false); } } diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index 56a25aa4dd9..e28bfb7b9c1 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -58,7 +58,7 @@ bool rseq_is_registered_for_current_thread(void); void -rseq_locate_rseq_regions(void); +rseq_locate_rseq_regions(bool saw_glibc_rseq_reg); void rseq_module_init(module_area_t *ma, bool at_map); diff --git a/suite/tests/CMakeLists.txt b/suite/tests/CMakeLists.txt index c4a52ba9308..1ea3fe5a0d4 100644 --- a/suite/tests/CMakeLists.txt +++ b/suite/tests/CMakeLists.txt @@ -5403,3 +5403,13 @@ if (BUILD_CLIENTS AND NOT ANDROID) code_api|tool.drcacheoff.simple PROPERTIES LABELS UBUNTU_22) endif () +if (LINUX AND X64 AND HAVE_RSEQ) + set_tests_properties( + code_api|api.rseq + code_api|linux.rseq_disable + code_api|linux.rseq_noarray + code_api|linux.rseq + code_api|linux.rseq_table + code_api|tool.drcacheoff.rseq + PROPERTIES LABELS UBUNTU_22) +endif () diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index 81cf72a9c93..e6e5650895b 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -58,6 +58,12 @@ # define _GNU_SOURCE #endif #include +#if defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 35)) +# include +# define GLIBC_RSEQ 1 +#else +# define RSEQ_SIG IF_X86_ELSE(0x90909090, 0xd503201f) /* nops to disasm nicely */ +#endif #include #include #undef NDEBUG @@ -66,7 +72,6 @@ #define EXPANDSTR(x) #x #define STRINGIFY(x) EXPANDSTR(x) -#define RSEQ_SIG IF_X86_ELSE(0x90909090, 0xd503201f) /* nops to disasm nicely */ #ifdef RSEQ_TEST_USE_OLD_SECTION_NAME # define RSEQ_SECTION_NAME "__rseq_table" #else @@ -107,6 +112,56 @@ static void *thread_ready; static volatile int sigill_count; +static bool +register_rseq() +{ +#ifdef GLIBC_RSEQ + if (__rseq_size > 0) { + /* Our glibc rseq handling in rseq_linux.c checks the following offset + * first for the glibc-registered struct rseq. Though we do have a + * fallback that does a wider search, it would be good to keep the + * expected offset in sync with glibc changes. + */ +# ifdef X86 +# ifdef X64 + assert(__rseq_offset == 2464); +# else + assert(__rseq_offset == 1312); +# endif +# else + assert(__rseq_offset == -32); +# endif + /* Already registered by glibc. Verify that it's there. */ + struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; + /* Glibc uses RSEQ_SIG as the signature. The following will return + * EBUSY if reg_rseq is the registered struct rseq and EINVAL otherwise. + * FTR: if we use some other value as signature (like zero), this + * becomes an EPERM vs EINVAL check. + */ + int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, RSEQ_SIG); + assert(res == -1 && (errno == EBUSY || errno == ENOSYS)); + } else { +#endif + rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; + int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); + assert(res == 0 || (res == -1 && errno == ENOSYS)); +#ifdef GLIBC_RSEQ + } +#endif + /* Linux kernel 4.18+ is required. */ + return errno != ENOSYS; +} + +static volatile struct rseq * +get_my_rseq() +{ +#ifdef GLIBC_RSEQ + if (__rseq_size > 0) + return (struct rseq *)(__builtin_thread_pointer() + __rseq_offset); +#endif + return &rseq_tls; +} + static void signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt) { @@ -117,6 +172,7 @@ signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt) static void test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_out) { + volatile struct rseq *reg_rseq = get_my_rseq(); /* We use static to avoid stack reference issues with our extra frame inside the asm. */ static __u32 id = RSEQ_CPU_ID_UNINITIALIZED; @@ -141,7 +197,7 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_simple(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test register inputs to the sequence. */ "movl %[cpu_id], %%eax\n\t" "movq %%rsp, %%rcx\n\t" @@ -193,12 +249,12 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), + : [cpu_id] "m"(reg_rseq->cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), [force_restart] "m"(force_restart) : "rax", "rcx", "rdx", "rbx", "rdi", "rsi", "memory"); #elif defined(AARCH64) @@ -218,7 +274,7 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o * inputs but that's simpler than manually computing all these * addresses inside the sequence. */ - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -262,12 +318,12 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), + : [cpu_id] "m"(reg_rseq->cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), [force_restart] "m"(force_restart) : "x0", "x1", "memory"); #else @@ -294,6 +350,7 @@ test_rseq_call(void) static void test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_out) { + volatile struct rseq *reg_rseq = get_my_rseq(); __u32 id = RSEQ_CPU_ID_UNINITIALIZED; int completions = 0; int restarts = 0; @@ -306,7 +363,7 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_branches(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "movl %[cpu_id], %%eax\n\t" /* Test "falling into" the rseq region. */ @@ -354,13 +411,13 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "13:\n\t" "12:\n\t" "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [force_restart] "m"(force_restart) + : [cpu_id] "m"(reg_rseq->cpu_id), [force_restart] "m"(force_restart) : "rax", "rcx", "memory"); #elif defined(AARCH64) __asm__ __volatile__( @@ -372,7 +429,7 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ /* Store the entry into the ptr. */ "adrp x0, rseq_cs_branches\n\t" "add x0, x0, :lo12:rseq_cs_branches\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -424,18 +481,20 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "13:\n\t" "12:\n\t" "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [force_restart] "m"(force_restart) + : [cpu_id] "m"(reg_rseq->cpu_id), [force_restart] "m"(force_restart) : "x0", "x1", "memory"); #else # error Unsupported arch #endif assert(id != RSEQ_CPU_ID_UNINITIALIZED); + *completions_out = completions; + *restarts_out = restarts; } static void @@ -461,6 +520,7 @@ test_rseq_branches(void) static void test_rseq_native_fault(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); int restarts = 0; #ifdef X86 __asm__ __volatile__( @@ -471,7 +531,7 @@ test_rseq_native_fault(void) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_fault(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" "pxor %%xmm0, %%xmm0\n\t" "mov $1,%%rcx\n\t" "movq %%rcx, %%xmm1\n\t" @@ -507,10 +567,10 @@ test_rseq_native_fault(void) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : : "rax", "rcx", "xmm0", "xmm1", "memory"); #elif defined(AARCH64) @@ -523,7 +583,7 @@ test_rseq_native_fault(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_fault\n\t" "add x0, x0, :lo12:rseq_cs_fault\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" "eor v0.16b, v0.16b, v0.16b\n\t" "mov x1, #1\n\t" "mov v1.D[0], x1\n\t" @@ -561,10 +621,10 @@ test_rseq_native_fault(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : : "x0", "x1", "q0", "q1", "memory"); #else @@ -585,6 +645,7 @@ test_rseq_native_fault(void) static void test_rseq_native_abort(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); #ifdef DEBUG /* See above: special code in core/ is DEBUG-only> */ int restarts = 0; # ifdef X86 @@ -596,7 +657,7 @@ test_rseq_native_abort(void) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_abort(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" "pxor %%xmm0, %%xmm0\n\t" "mov $1,%%rcx\n\t" "movq %%rcx, %%xmm1\n\t" @@ -647,13 +708,13 @@ test_rseq_native_abort(void) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : [cpu_mask_size] "i"(sizeof(cpu_set_t)), [sysnum_setaffinity] "i"(SYS_sched_setaffinity) - : "rax", "rcx", "rdx", "xmm0", "xmm1", "memory"); + : "rax", "rcx", "rdx", "rsi", "rdi", "xmm0", "xmm1", "memory"); # elif defined(AARCH64) __asm__ __volatile__( /* clang-format off */ /* (avoid indenting next few lines) */ @@ -664,7 +725,7 @@ test_rseq_native_abort(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_abort\n\t" "add x0, x0, :lo12:rseq_cs_abort\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" "eor v0.16b, v0.16b, v0.16b\n\t" "mov x1, #1\n\t" "mov v1.D[0], x1\n\t" @@ -720,10 +781,10 @@ test_rseq_native_abort(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : [cpu_mask_size] "i"(sizeof(cpu_set_t)), [sysnum_setaffinity] "i"(SYS_sched_setaffinity) : "x0", "x1", "x2", "x8", "q0", "q1", "memory"); @@ -739,6 +800,7 @@ test_rseq_native_abort(void) static void test_rseq_writeback_store(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); __u32 id = RSEQ_CPU_ID_UNINITIALIZED; int array[2] = { 0 }; int *index = array; @@ -751,7 +813,7 @@ test_rseq_writeback_store(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_writeback\n\t" "add x0, x0, :lo12:rseq_cs_writeback\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -784,11 +846,11 @@ test_rseq_writeback_store(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), [index] "=g"(index) - : [cpu_id] "m"(rseq_tls.cpu_id) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [index] "=g"(index) + : [cpu_id] "m"(reg_rseq->cpu_id) : "x0", "x1", "x2", "x28", "memory"); assert(id != RSEQ_CPU_ID_UNINITIALIZED); } @@ -802,10 +864,9 @@ rseq_thread_loop(void *arg) * in this function is close enough: the test already has non-determinism. */ signal_cond_var(thread_ready); - rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; - int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); - if (res != 0) - return NULL; + bool res = register_rseq(); + assert(res); + volatile struct rseq *reg_rseq = get_my_rseq(); static int zero; # ifdef X86 __asm__ __volatile__( @@ -816,7 +877,7 @@ rseq_thread_loop(void *arg) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_thread(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test "falling into" the rseq region. */ /* Restartable sequence. We loop to ensure we're in the region on @@ -851,10 +912,10 @@ rseq_thread_loop(void *arg) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [zero] "=m"(zero) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [zero] "=m"(zero) : [exit_requested] "m"(exit_requested) : "rax", "memory"); # elif defined(AARCH64) @@ -867,7 +928,7 @@ rseq_thread_loop(void *arg) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_thread\n\t" "add x0, x0, :lo12:rseq_cs_thread\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test "falling into" the rseq region. */ /* Restartable sequence. We loop to ensure we're in the region on @@ -902,10 +963,10 @@ rseq_thread_loop(void *arg) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [zero] "=m"(zero) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [zero] "=m"(zero) : [exit_requested] "m"(exit_requested) : "x0", "x1", "memory"); # else @@ -963,7 +1024,7 @@ kernel_xfer_event(void *drcontext, const dr_kernel_xfer_info_t *info) DR_EXPORT void dr_client_main(client_id_t id, int argc, const char *argv[]) { - /* Ensure DR_XFER_RSEQ_ABORT is rasied. */ + /* Ensure DR_XFER_RSEQ_ABORT is raised. */ dr_register_kernel_xfer_event(kernel_xfer_event); drmemtrace_client_main(id, argc, argv); } @@ -973,9 +1034,7 @@ int main() { intercept_signal(SIGILL, signal_handler, false); - rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; - int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); - if (res == 0) { + if (register_rseq()) { #ifdef RSEQ_TEST_ATTACH /* Set -offline to avoid trying to open a pipe to a missing reader. */ if (setenv("DYNAMORIO_OPTIONS", "-stderr_mask 0xc -client_lib ';;-offline'", @@ -1011,9 +1070,6 @@ main() join_thread(mythread); destroy_cond_var(thread_ready); #endif - } else { - /* Linux kernel 4.18+ is required. */ - assert(errno == ENOSYS); } print("All done\n"); return 0; diff --git a/tools/drdeploy.c b/tools/drdeploy.c index 8f751931895..6d82b8a5ddd 100644 --- a/tools/drdeploy.c +++ b/tools/drdeploy.c @@ -1282,20 +1282,6 @@ _tmain(int argc, TCHAR *targv[]) /* we re-read the tool list if the root, platform or toolconfig dir change */ read_tool_list(dr_toolconfig_dir, dr_platform); -#if defined(LINUX) && !defined(ANDROID) && (defined(DRCONFIG) || defined(DRRUN)) - /* XXX i#5431: Workaround for an rseq issue with glibc 2.35 which makes many - * apps fail up front. We expect to remove this once the real fix is in place. - */ -# include - const char *libc_ver = gnu_get_libc_version(); - if (libc_ver[0] != '\0' && libc_ver[0] >= '2' && libc_ver[1] == '.' && - libc_ver[2] >= '3' && libc_ver[3] >= '5') { - info("glibc2.35 detected: setting -disable_rseq"); - add_extra_option(extra_ops, BUFFER_SIZE_ELEMENTS(extra_ops), &extra_ops_sofar, - "-disable_rseq"); - } -#endif - /* parse command line */ for (i = 1; i < argc; i++) {