From 0115daa9bf9947262a4cc61a3936b44cf91d674a Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 28 Oct 2022 16:14:12 -0400 Subject: [PATCH 01/36] i#5431: Support GLIBC rseq support Issue: #5431 --- core/arch/interp.c | 12 ++++ core/dynamo.c | 1 - core/unix/os.c | 5 ++ core/unix/rseq_linux.c | 95 ++++++++++++++++++++++++++- core/unix/rseq_linux.h | 7 ++ suite/tests/linux/rseq.c | 135 ++++++++++++++++++++++++++------------- tools/drdeploy.c | 14 ---- 7 files changed, 206 insertions(+), 63 deletions(-) diff --git a/core/arch/interp.c b/core/arch/interp.c index fa51eef80c5..7667a772cb2 100644 --- a/core/arch/interp.c +++ b/core/arch/interp.c @@ -66,6 +66,9 @@ #include "../native_exec.h" #include "../jit_opt.h" +#if !defined(STATIC_LIBRARY) && defined(LINUX) +# include "../unix/rseq_linux.h" +#endif #ifdef CHECK_RETURNS_SSE2 # include /* for warning when see libc setjmp */ #endif @@ -373,6 +376,15 @@ set_reached_image_entry() SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); reached_image_entry = true; SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); +#if !defined(STATIC_LIBRARY) && defined(LINUX) + /* For enabling rseq on non-static DR, we need to wait till libc is + * initialized. This is a good control point. + * Note that d_r_rseq_init would already have completed by now. + */ + rseq_check_glibc_enabled(); + if (rseq_is_registered_for_current_thread()) + rseq_locate_rseq_regions(); +#endif } bool diff --git a/core/dynamo.c b/core/dynamo.c index d534321d32b..d09a4559fd0 100644 --- a/core/dynamo.c +++ b/core/dynamo.c @@ -692,7 +692,6 @@ dynamorio_app_init_part_two_finalize(void) find_dynamo_library_vm_areas(); dynamo_vm_areas_unlock(); } - #ifdef ANNOTATIONS annotation_init(); #endif diff --git a/core/unix/os.c b/core/unix/os.c index ffb1f5649d6..6a180d064ef 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -10075,6 +10075,11 @@ find_executable_vm_areas(void) /* now that we have the modules set up, query libc */ get_libc_errno_location(true /*force init*/); +#if !defined(STATIC_LIBRARY) && defined(LINUX) + /* For rseq on GLIBC >= 2.35, we need to find __rseq_size and __rseq_offset, + * but we cannot do that here as they wouldn't be initialized yet. + */ +#endif return count; } diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index c0c691644e9..b8357dbf935 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -50,7 +50,6 @@ #include #include "include/syscall.h" #include - /* The linux/rseq.h header made a source-breaking change in torvalds/linux@bfdf4e6 * which broke our build. To avoid future issues we use our own definitions. * Binary breakage is unlikely without long periods of deprecation so this is @@ -73,6 +72,10 @@ struct rseq { } __attribute__((aligned(4 * sizeof(uint64)))); #define RSEQ_FLAG_UNREGISTER 1 +#ifdef GLIBC_RSEQ +static volatile bool glibc_rseq_enabled; +#endif + vm_area_vector_t *d_r_rseq_areas; DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, INIT_LOCK_FREE(rseq_trigger_lock)); @@ -151,6 +154,66 @@ rseq_cs_free(dcontext_t *dcontext, void *data) } while (record != NULL); } +void +rseq_check_glibc_enabled() +{ +#ifdef GLIBC_RSEQ + ASSERT(!glibc_rseq_enabled); +# ifndef DEBUG + /* Already found rseq_tls_offset. No need to keep looking. */ + if (rseq_tls_offset != 0) + return; +# else + /* Even though rseq_tls_offset may already have been found, we still + * want to find it again to make sure all modules exporting + * __rseq_offset still have the same value. + */ +# endif + module_iterator_t *iter = module_iterator_start(); + while (module_iterator_hasnext(iter)) { + module_area_t *ma = module_iterator_next(iter); + const char *modname = GET_MODULE_NAME(&ma->names); + if (modname == NULL || strstr(modname, "libc.so") != modname) + continue; + uint *rseq_size_addr = + (uint *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_size"); + // if (rseq_size_addr != NULL ) { + // dr_printf("AAA found rseq_size for %s\n",modname); + // } + // ASSERT(rseq_size_addr != NULL); + /* __rseq_size is set to zero if the rseq support in glibc is disabled + * by exporting GLIBC_TUNABLES=glibc.pthread.rseq=0. Or it's possible + * that we're using early inject and it just hasn't been initialized + * yet. For STATIC_LIBRARY, the caller needs to call this routine just + * one time, during d_r_rseq_init. + */ + if (rseq_size_addr == NULL || *rseq_size_addr == 0) + continue; + int *rseq_offset_addr = + (int *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_offset"); + ASSERT(rseq_offset_addr != NULL); + if (rseq_tls_offset == 0) { + rseq_tls_offset = *rseq_offset_addr; + bool new_value = true; + ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); + DODEBUG({ + byte *rseq_addr = get_app_segment_base(LIB_SEG_TLS) + rseq_tls_offset; + LOG(GLOBAL, LOG_LOADER, 2, + "Found struct rseq registered by glibc @ " PFX " at TLS offset %d\n", + rseq_addr, rseq_tls_offset); + int res = + dynamorio_syscall(SYS_rseq, 4, rseq_addr, sizeof(struct rseq), 0, 0); + ASSERT(res == -EPERM); + }); + } else { + ASSERT(rseq_tls_offset == *rseq_offset_addr); + ASSERT(glibc_rseq_enabled); + } + } + module_iterator_stop(iter); +#endif +} + void d_r_rseq_init(void) { @@ -161,9 +224,16 @@ d_r_rseq_init(void) rseq_cs_table = generic_hash_create(GLOBAL_DCONTEXT, INIT_RSEQ_CS_TABLE_SIZE, 80, HASHTABLE_SHARED | HASHTABLE_PERSISTENT, rseq_cs_free _IF_DEBUG("rseq_cs table")); +#ifdef STATIC_LIBRARY + rseq_check_glibc_enabled(); /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ if (rseq_is_registered_for_current_thread()) rseq_locate_rseq_regions(); +#else + /* For non-static DR (early-inject), we delay enabling rseq till libc is + * completely initialized. + */ +#endif } void @@ -172,6 +242,10 @@ d_r_rseq_exit(void) generic_hash_destroy(GLOBAL_DCONTEXT, rseq_cs_table); vmvector_delete_vector(GLOBAL_DCONTEXT, d_r_rseq_areas); DELETE_LOCK(rseq_trigger_lock); + /* TODO(abnv): For reattaches, maybe we should retain these. */ + bool new_value = false; + ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); + rseq_tls_offset = 0; } void @@ -229,7 +303,7 @@ static void rseq_clear_tls_ptr(dcontext_t *dcontext) { ASSERT(rseq_tls_offset != 0); - byte *base = get_segment_base(LIB_SEG_TLS); + byte *base = get_app_segment_base(LIB_SEG_TLS); struct rseq *app_rseq = (struct rseq *)(base + rseq_tls_offset); /* We're directly writing this in the cache, so we do not bother with safe_read * or safe_write here either. We already cannot handle rseq adversarial cases. @@ -310,6 +384,8 @@ rseq_shared_fragment_flushtime_update(dcontext_t *dcontext) bool rseq_is_registered_for_current_thread(void) { + if (glibc_rseq_enabled) + return true; /* Unfortunately there's no way to query the current rseq struct. * For 64-bit we can pass a kernel address and look for EFAULT * vs EINVAL, but there is no kernel address for 32-bit. @@ -627,6 +703,10 @@ rseq_process_module(module_area_t *ma, bool at_map) static int rseq_locate_tls_offset(void) { + if (glibc_rseq_enabled) { + ASSERT(rseq_tls_offset != 0); + return rseq_tls_offset; + } /* We assume (and document) that the loader's static TLS is used, so every thread * has a consistent %fs:-offs address. Unfortunately, using a local copy of the * rseq code for our non-instrumented execution requires us to locate the app's @@ -727,6 +807,14 @@ rseq_locate_rseq_regions(void) { if (rseq_enabled) return; +#ifdef GLIBC_RSEQ + /* We delay locating struct rseq TLS offset and rseq_cs until GLIBC is + * completely initialized. + */ + if (!reached_image_entry_yet()) { + return; + } +#endif /* This is a global operation, but the trigger could be hit by two threads at once, * thus requiring synchronization. */ @@ -768,6 +856,9 @@ rseq_locate_rseq_regions(void) void rseq_module_init(module_area_t *ma, bool at_map) { + /* If !rseq_enabled, the module will simply be processed when we enable + * rseq. + */ if (rseq_enabled) { rseq_process_module(ma, at_map); } diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index 56a25aa4dd9..d33ef70dda8 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -45,6 +45,10 @@ # error Rseq is a Linux-only feature #endif +#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 35) +# define GLIBC_RSEQ +#endif + void d_r_rseq_init(void); @@ -66,4 +70,7 @@ rseq_module_init(module_area_t *ma, bool at_map); void rseq_process_syscall(dcontext_t *dcontext); +void +rseq_check_glibc_enabled(void); + #endif /* _RSEQ_H_ */ diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index fe4d22e2aa0..086231c9380 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -58,6 +58,12 @@ # define _GNU_SOURCE #endif #include +#if defined(__GLIBC__) && ((__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 35)) +# include +# define GLIBC_RSEQ 1 +#else +# define RSEQ_SIG IF_X86_ELSE(0x90909090, 0xd503201f) /* nops to disasm nicely */ +#endif #include #include #undef NDEBUG @@ -66,7 +72,6 @@ #define EXPANDSTR(x) #x #define STRINGIFY(x) EXPANDSTR(x) -#define RSEQ_SIG IF_X86_ELSE(0x90909090, 0xd503201f) /* nops to disasm nicely */ #ifdef RSEQ_TEST_USE_OLD_SECTION_NAME # define RSEQ_SECTION_NAME "__rseq_table" #else @@ -90,6 +95,11 @@ # define RSEQ_ADD_ARRAY_ENTRY(label) /* Nothing. */ #endif +static int rseq_tls_offset; + +#ifdef GLIBC_RSEQ +static bool glibc_rseq_enabled; +#endif /* This cannot be a stack-local variable, as the kernel will force SIGSEGV * if it can't read this struct. And for multiple threads it should be in TLS. */ @@ -107,6 +117,16 @@ static void *thread_ready; static volatile int sigill_count; +static volatile struct rseq * +get_my_rseq() +{ +#ifdef GLIBC_RSEQ + if (glibc_rseq_enabled) + return (struct rseq *)(__builtin_thread_pointer() + rseq_tls_offset); +#endif + return &rseq_tls; +} + static void signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt) { @@ -117,6 +137,7 @@ signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt) static void test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_out) { + volatile struct rseq *reg_rseq = get_my_rseq(); /* We use static to avoid stack reference issues with our extra frame inside the asm. */ static __u32 id = RSEQ_CPU_ID_UNINITIALIZED; @@ -141,7 +162,7 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_simple(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test register inputs to the sequence. */ "movl %[cpu_id], %%eax\n\t" "movq %%rsp, %%rcx\n\t" @@ -193,12 +214,12 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), + : [cpu_id] "m"(reg_rseq->cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), [force_restart] "m"(force_restart) : "rax", "rcx", "rdx", "rbx", "rdi", "rsi", "memory"); #elif defined(AARCH64) @@ -218,7 +239,7 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o * inputs but that's simpler than manually computing all these * addresses inside the sequence. */ - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -258,12 +279,12 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), + : [cpu_id] "m"(reg_rseq->cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED), [force_restart] "m"(force_restart) : "x0", "x1", "memory"); #else @@ -290,6 +311,7 @@ test_rseq_call(void) static void test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_out) { + volatile struct rseq *reg_rseq = get_my_rseq(); __u32 id = RSEQ_CPU_ID_UNINITIALIZED; int completions = 0; int restarts = 0; @@ -302,7 +324,7 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_branches(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "movl %[cpu_id], %%eax\n\t" /* Test "falling into" the rseq region. */ @@ -350,13 +372,13 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "13:\n\t" "12:\n\t" "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [force_restart] "m"(force_restart) + : [cpu_id] "m"(reg_rseq->cpu_id), [force_restart] "m"(force_restart) : "rax", "rcx", "memory"); #elif defined(AARCH64) __asm__ __volatile__( @@ -368,7 +390,7 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ /* Store the entry into the ptr. */ "adrp x0, rseq_cs_branches\n\t" "add x0, x0, :lo12:rseq_cs_branches\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -420,18 +442,20 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_ "13:\n\t" "12:\n\t" "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [completions] "=m"(completions), [restarts] "=m"(restarts), [force_restart_write] "=m"(force_restart) - : [cpu_id] "m"(rseq_tls.cpu_id), [force_restart] "m"(force_restart) + : [cpu_id] "m"(reg_rseq->cpu_id), [force_restart] "m"(force_restart) : "x0", "x1", "memory"); #else # error Unsupported arch #endif assert(id != RSEQ_CPU_ID_UNINITIALIZED); + *completions_out = completions; + *restarts_out = restarts; } static void @@ -457,6 +481,7 @@ test_rseq_branches(void) static void test_rseq_native_fault(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); int restarts = 0; #ifdef X86 __asm__ __volatile__( @@ -467,7 +492,7 @@ test_rseq_native_fault(void) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_fault(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" "pxor %%xmm0, %%xmm0\n\t" "mov $1,%%rcx\n\t" "movq %%rcx, %%xmm1\n\t" @@ -503,10 +528,10 @@ test_rseq_native_fault(void) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : : "rax", "rcx", "xmm0", "xmm1", "memory"); #elif defined(AARCH64) @@ -519,7 +544,7 @@ test_rseq_native_fault(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_fault\n\t" "add x0, x0, :lo12:rseq_cs_fault\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" "eor v0.16b, v0.16b, v0.16b\n\t" "mov x1, #1\n\t" "mov v1.D[0], x1\n\t" @@ -557,10 +582,10 @@ test_rseq_native_fault(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : : "x0", "x1", "q0", "q1", "memory"); #else @@ -581,6 +606,7 @@ test_rseq_native_fault(void) static void test_rseq_native_abort(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); #ifdef DEBUG /* See above: special code in core/ is DEBUG-only> */ int restarts = 0; # ifdef X86 @@ -592,7 +618,7 @@ test_rseq_native_abort(void) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_abort(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" "pxor %%xmm0, %%xmm0\n\t" "mov $1,%%rcx\n\t" "movq %%rcx, %%xmm1\n\t" @@ -643,13 +669,13 @@ test_rseq_native_abort(void) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : [cpu_mask_size] "i"(sizeof(cpu_set_t)), [sysnum_setaffinity] "i"(SYS_sched_setaffinity) - : "rax", "rcx", "rdx", "xmm0", "xmm1", "memory"); + : "rax", "rcx", "rdx", "rsi", "rdi", "xmm0", "xmm1", "memory"); # elif defined(AARCH64) __asm__ __volatile__( /* clang-format off */ /* (avoid indenting next few lines) */ @@ -660,7 +686,7 @@ test_rseq_native_abort(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_abort\n\t" "add x0, x0, :lo12:rseq_cs_abort\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" "eor v0.16b, v0.16b, v0.16b\n\t" "mov x1, #1\n\t" "mov v1.D[0], x1\n\t" @@ -716,10 +742,10 @@ test_rseq_native_abort(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [restarts] "=m"(restarts) : [cpu_mask_size] "i"(sizeof(cpu_set_t)), [sysnum_setaffinity] "i"(SYS_sched_setaffinity) : "x0", "x1", "x2", "x8", "q0", "q1", "memory"); @@ -735,6 +761,7 @@ test_rseq_native_abort(void) static void test_rseq_writeback_store(void) { + volatile struct rseq *reg_rseq = get_my_rseq(); __u32 id = RSEQ_CPU_ID_UNINITIALIZED; int array[2] = { 0 }; int *index = array; @@ -747,7 +774,7 @@ test_rseq_writeback_store(void) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_writeback\n\t" "add x0, x0, :lo12:rseq_cs_writeback\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test a register input to the sequence. */ "ldr x0, %[cpu_id]\n\t" /* Test "falling into" the rseq region. */ @@ -780,11 +807,11 @@ test_rseq_writeback_store(void) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), [index] "=g"(index) - : [cpu_id] "m"(rseq_tls.cpu_id) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [id] "=m"(id), [index] "=g"(index) + : [cpu_id] "m"(reg_rseq->cpu_id) : "x0", "x1", "x2", "x28", "memory"); assert(id != RSEQ_CPU_ID_UNINITIALIZED); } @@ -798,8 +825,9 @@ rseq_thread_loop(void *arg) * in this function is close enough: the test already has non-determinism. */ signal_cond_var(thread_ready); - rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; - int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); + volatile struct rseq *reg_rseq = get_my_rseq(); + reg_rseq->cpu_id = RSEQ_CPU_ID_UNINITIALIZED; + int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, RSEQ_SIG); if (res != 0) return NULL; static int zero; @@ -812,7 +840,7 @@ rseq_thread_loop(void *arg) "6:\n\t" /* Store the entry into the ptr. */ "leaq rseq_cs_thread(%%rip), %%rax\n\t" - "movq %%rax, %[rseq_tls]\n\t" + "movq %%rax, %[rseq_cs]\n\t" /* Test "falling into" the rseq region. */ /* Restartable sequence. We loop to ensure we're in the region on @@ -847,10 +875,10 @@ rseq_thread_loop(void *arg) /* Clear the ptr. */ "5:\n\t" - "movq $0, %[rseq_tls]\n\t" + "movq $0, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [zero] "=m"(zero) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [zero] "=m"(zero) : [exit_requested] "m"(exit_requested) : "rax", "memory"); # elif defined(AARCH64) @@ -863,7 +891,7 @@ rseq_thread_loop(void *arg) /* Store the entry into the ptr. */ "adrp x0, rseq_cs_thread\n\t" "add x0, x0, :lo12:rseq_cs_thread\n\t" - "str x0, %[rseq_tls]\n\t" + "str x0, %[rseq_cs]\n\t" /* Test "falling into" the rseq region. */ /* Restartable sequence. We loop to ensure we're in the region on @@ -898,10 +926,10 @@ rseq_thread_loop(void *arg) /* Clear the ptr. */ "5:\n\t" - "str xzr, %[rseq_tls]\n\t" + "str xzr, %[rseq_cs]\n\t" /* clang-format on */ - : [rseq_tls] "=m"(rseq_tls.rseq_cs), [zero] "=m"(zero) + : [rseq_cs] "=m"(reg_rseq->rseq_cs), [zero] "=m"(zero) : [exit_requested] "m"(exit_requested) : "x0", "x1", "memory"); # else @@ -959,7 +987,7 @@ kernel_xfer_event(void *drcontext, const dr_kernel_xfer_info_t *info) DR_EXPORT void dr_client_main(client_id_t id, int argc, const char *argv[]) { - /* Ensure DR_XFER_RSEQ_ABORT is rasied. */ + /* Ensure DR_XFER_RSEQ_ABORT is raised. */ dr_register_kernel_xfer_event(kernel_xfer_event); drmemtrace_client_main(id, argc, argv); } @@ -969,9 +997,24 @@ int main() { intercept_signal(SIGILL, signal_handler, false); - rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; - int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); - if (res == 0) { + int res; + int expected_errno; +#ifdef GLIBC_RSEQ + glibc_rseq_enabled = __rseq_size > 0; + if (glibc_rseq_enabled) { + assert(__rseq_offset > 0); + struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; + res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, 0); + expected_errno = EPERM; + } else { +#endif + rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; + res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); + expected_errno = 0; +#ifdef GLIBC_RSEQ + } +#endif + if (errno == expected_errno) { #ifdef RSEQ_TEST_ATTACH /* Set -offline to avoid trying to open a pipe to a missing reader. */ if (setenv("DYNAMORIO_OPTIONS", "-stderr_mask 0xc -client_lib ';;-offline'", diff --git a/tools/drdeploy.c b/tools/drdeploy.c index 8f751931895..6d82b8a5ddd 100644 --- a/tools/drdeploy.c +++ b/tools/drdeploy.c @@ -1282,20 +1282,6 @@ _tmain(int argc, TCHAR *targv[]) /* we re-read the tool list if the root, platform or toolconfig dir change */ read_tool_list(dr_toolconfig_dir, dr_platform); -#if defined(LINUX) && !defined(ANDROID) && (defined(DRCONFIG) || defined(DRRUN)) - /* XXX i#5431: Workaround for an rseq issue with glibc 2.35 which makes many - * apps fail up front. We expect to remove this once the real fix is in place. - */ -# include - const char *libc_ver = gnu_get_libc_version(); - if (libc_ver[0] != '\0' && libc_ver[0] >= '2' && libc_ver[1] == '.' && - libc_ver[2] >= '3' && libc_ver[3] >= '5') { - info("glibc2.35 detected: setting -disable_rseq"); - add_extra_option(extra_ops, BUFFER_SIZE_ELEMENTS(extra_ops), &extra_ops_sofar, - "-disable_rseq"); - } -#endif - /* parse command line */ for (i = 1; i < argc; i++) { From c6ac74d06665f904fb95013c8ae0e82041783cd2 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 28 Oct 2022 17:08:12 -0400 Subject: [PATCH 02/36] Some fixes. --- core/dynamo.c | 14 ++++++++++++++ core/unix/os.c | 8 -------- core/unix/rseq_linux.c | 23 ++++++++++++----------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/core/dynamo.c b/core/dynamo.c index d09a4559fd0..d54552b6ce6 100644 --- a/core/dynamo.c +++ b/core/dynamo.c @@ -92,6 +92,9 @@ portable and to avoid frequency scaling." # endif #endif +#ifdef LINUX +# include "rseq_linux.h" +#endif /* global thread-shared variables */ bool dynamo_initialized = false; static bool dynamo_options_initialized = false; @@ -692,6 +695,13 @@ dynamorio_app_init_part_two_finalize(void) find_dynamo_library_vm_areas(); dynamo_vm_areas_unlock(); } +#ifdef LINUX + /* We need to delay initializing rseq support to after we have + * initialized modules. + */ + if (!standalone_library) + d_r_rseq_init(); +#endif #ifdef ANNOTATIONS annotation_init(); #endif @@ -1167,6 +1177,10 @@ dynamo_shared_exit(thread_record_t *toexit /* must ==cur thread for Linux */ os_fast_exit(); os_slow_exit(); native_exec_exit(); /* before vm_areas_exit for using dynamo_areas */ +#ifdef LINUX + if (!standalone_library) + d_r_rseq_exit(); +#endif vm_areas_exit(); perscache_slow_exit(); /* fast called in dynamo_process_exit_with_thread_info() */ modules_exit(); /* after aslr_exit() from os_slow_exit(), diff --git a/core/unix/os.c b/core/unix/os.c index 6a180d064ef..161749af328 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -1031,10 +1031,6 @@ d_r_os_init(void) */ init_android_version(); #endif -#ifdef LINUX - if (!standalone_library) - d_r_rseq_init(); -#endif #ifdef MACOS64 tls_process_init(); #endif @@ -1441,10 +1437,6 @@ os_slow_exit(void) { #ifdef MACOS64 tls_process_exit(); -#endif -#ifdef LINUX - if (!standalone_library) - d_r_rseq_exit(); #endif d_r_signal_exit(); memquery_exit(); diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index b8357dbf935..a5450ae5735 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -82,7 +82,7 @@ DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, static volatile bool rseq_enabled; /* We require all threads to use the same TLS offset to point at struct rseq. */ -static int rseq_tls_offset; +static volatile int rseq_tls_offset; /* The signature is registered per thread, but we require all registrations * to be the same. @@ -159,6 +159,11 @@ rseq_check_glibc_enabled() { #ifdef GLIBC_RSEQ ASSERT(!glibc_rseq_enabled); + #ifdef STATIC_LIBRARY + ASSERT(rseq_tls_offset == 0); + #else + /* We may already have initialized rseq_tls_offset in rseq_process_syscall*/ + #endif # ifndef DEBUG /* Already found rseq_tls_offset. No need to keep looking. */ if (rseq_tls_offset != 0) @@ -172,15 +177,8 @@ rseq_check_glibc_enabled() module_iterator_t *iter = module_iterator_start(); while (module_iterator_hasnext(iter)) { module_area_t *ma = module_iterator_next(iter); - const char *modname = GET_MODULE_NAME(&ma->names); - if (modname == NULL || strstr(modname, "libc.so") != modname) - continue; uint *rseq_size_addr = (uint *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_size"); - // if (rseq_size_addr != NULL ) { - // dr_printf("AAA found rseq_size for %s\n",modname); - // } - // ASSERT(rseq_size_addr != NULL); /* __rseq_size is set to zero if the rseq support in glibc is disabled * by exporting GLIBC_TUNABLES=glibc.pthread.rseq=0. Or it's possible * that we're using early inject and it just hasn't been initialized @@ -191,9 +189,9 @@ rseq_check_glibc_enabled() continue; int *rseq_offset_addr = (int *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_offset"); - ASSERT(rseq_offset_addr != NULL); + ASSERT(rseq_offset_addr != NULL);; if (rseq_tls_offset == 0) { - rseq_tls_offset = *rseq_offset_addr; + ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); bool new_value = true; ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); DODEBUG({ @@ -207,7 +205,9 @@ rseq_check_glibc_enabled() }); } else { ASSERT(rseq_tls_offset == *rseq_offset_addr); +# ifdef STATIC_LIBRARY ASSERT(glibc_rseq_enabled); +# endif } } module_iterator_stop(iter); @@ -245,7 +245,8 @@ d_r_rseq_exit(void) /* TODO(abnv): For reattaches, maybe we should retain these. */ bool new_value = false; ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); - rseq_tls_offset = 0; + uint new_offset = 0; + ATOMIC_4BYTE_WRITE(&rseq_tls_offset, new_offset, false); } void From d07698f2419c60eeb66be6d426d329357bf21a89 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 28 Oct 2022 17:47:51 -0400 Subject: [PATCH 03/36] Fix api.rseq --- core/unix/rseq_linux.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index a5450ae5735..701e6ac4303 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -159,11 +159,14 @@ rseq_check_glibc_enabled() { #ifdef GLIBC_RSEQ ASSERT(!glibc_rseq_enabled); - #ifdef STATIC_LIBRARY +# ifdef STATIC_LIBRARY ASSERT(rseq_tls_offset == 0); - #else - /* We may already have initialized rseq_tls_offset in rseq_process_syscall*/ - #endif +# else + /* For !STATIC_LIBRARY we delay rseq_check_glibc_enabled until we GLIBC is + * initialized, so it may happen that rseq_tls_offset is already initialized + * by rseq_process_syscall. + */ +# endif # ifndef DEBUG /* Already found rseq_tls_offset. No need to keep looking. */ if (rseq_tls_offset != 0) @@ -189,7 +192,7 @@ rseq_check_glibc_enabled() continue; int *rseq_offset_addr = (int *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_offset"); - ASSERT(rseq_offset_addr != NULL);; + ASSERT(rseq_offset_addr != NULL); if (rseq_tls_offset == 0) { ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); bool new_value = true; @@ -808,7 +811,7 @@ rseq_locate_rseq_regions(void) { if (rseq_enabled) return; -#ifdef GLIBC_RSEQ +#if defined(GLIBC_RSEQ) && !defined(STATIC_LIBRARY) /* We delay locating struct rseq TLS offset and rseq_cs until GLIBC is * completely initialized. */ From 3b0805d8016471387f0f6555ecda9a79133aa99b Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 28 Oct 2022 20:26:58 -0400 Subject: [PATCH 04/36] Add missing conditional compilation. --- core/unix/rseq_linux.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 701e6ac4303..71f33222dfd 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -245,9 +245,10 @@ d_r_rseq_exit(void) generic_hash_destroy(GLOBAL_DCONTEXT, rseq_cs_table); vmvector_delete_vector(GLOBAL_DCONTEXT, d_r_rseq_areas); DELETE_LOCK(rseq_trigger_lock); - /* TODO(abnv): For reattaches, maybe we should retain these. */ +#ifdef GLIBC_RSEQ bool new_value = false; ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); +#endif uint new_offset = 0; ATOMIC_4BYTE_WRITE(&rseq_tls_offset, new_offset, false); } @@ -388,8 +389,10 @@ rseq_shared_fragment_flushtime_update(dcontext_t *dcontext) bool rseq_is_registered_for_current_thread(void) { +#ifdef GLIBC_RSEQ if (glibc_rseq_enabled) return true; +#endif /* Unfortunately there's no way to query the current rseq struct. * For 64-bit we can pass a kernel address and look for EFAULT * vs EINVAL, but there is no kernel address for 32-bit. @@ -707,10 +710,12 @@ rseq_process_module(module_area_t *ma, bool at_map) static int rseq_locate_tls_offset(void) { +#ifdef GLIBC_RSEQ if (glibc_rseq_enabled) { ASSERT(rseq_tls_offset != 0); return rseq_tls_offset; } +#endif /* We assume (and document) that the loader's static TLS is used, so every thread * has a consistent %fs:-offs address. Unfortunately, using a local copy of the * rseq code for our non-instrumented execution requires us to locate the app's From a88e7d79b0fbcc65d16d5ab1725380fff0e9afd6 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Sat, 29 Oct 2022 00:40:14 -0400 Subject: [PATCH 05/36] Cleanup. --- core/unix/rseq_linux.c | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 71f33222dfd..acf42777674 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -166,16 +166,6 @@ rseq_check_glibc_enabled() * initialized, so it may happen that rseq_tls_offset is already initialized * by rseq_process_syscall. */ -# endif -# ifndef DEBUG - /* Already found rseq_tls_offset. No need to keep looking. */ - if (rseq_tls_offset != 0) - return; -# else - /* Even though rseq_tls_offset may already have been found, we still - * want to find it again to make sure all modules exporting - * __rseq_offset still have the same value. - */ # endif module_iterator_t *iter = module_iterator_start(); while (module_iterator_hasnext(iter)) { @@ -185,8 +175,7 @@ rseq_check_glibc_enabled() /* __rseq_size is set to zero if the rseq support in glibc is disabled * by exporting GLIBC_TUNABLES=glibc.pthread.rseq=0. Or it's possible * that we're using early inject and it just hasn't been initialized - * yet. For STATIC_LIBRARY, the caller needs to call this routine just - * one time, during d_r_rseq_init. + * yet. */ if (rseq_size_addr == NULL || *rseq_size_addr == 0) continue; @@ -234,7 +223,7 @@ d_r_rseq_init(void) rseq_locate_rseq_regions(); #else /* For non-static DR (early-inject), we delay enabling rseq till libc is - * completely initialized. + * completely initialized, in set_reached_image_entry. */ #endif } From 1b8a871b9d735a0fff8c9956d15660d5030ff611 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Sat, 29 Oct 2022 00:40:28 -0400 Subject: [PATCH 06/36] Add tmate --- .github/workflows/ci-x86.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci-x86.yml b/.github/workflows/ci-x86.yml index f2b4da11f8b..11416c64f50 100644 --- a/.github/workflows/ci-x86.yml +++ b/.github/workflows/ci-x86.yml @@ -172,6 +172,13 @@ jobs: CI_TRIGGER: ${{ github.event_name }} CI_BRANCH: ${{ github.ref }} + - name: Setup tmate session on failure + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + + - name: Setup tmate session on success + uses: mxschmitt/action-tmate@v3 + - name: Send failure mail to dynamorio-devs if: failure() && github.ref == 'refs/heads/master' uses: dawidd6/action-send-mail@v2 From 0a0dfce465d2ae59988fdb1005146bb9d83a0321 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Sat, 29 Oct 2022 08:57:24 -0400 Subject: [PATCH 07/36] Revert "Add tmate" This reverts commit 1b8a871b9d735a0fff8c9956d15660d5030ff611. --- .github/workflows/ci-x86.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci-x86.yml b/.github/workflows/ci-x86.yml index 11416c64f50..f2b4da11f8b 100644 --- a/.github/workflows/ci-x86.yml +++ b/.github/workflows/ci-x86.yml @@ -172,13 +172,6 @@ jobs: CI_TRIGGER: ${{ github.event_name }} CI_BRANCH: ${{ github.ref }} - - name: Setup tmate session on failure - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - - - name: Setup tmate session on success - uses: mxschmitt/action-tmate@v3 - - name: Send failure mail to dynamorio-devs if: failure() && github.ref == 'refs/heads/master' uses: dawidd6/action-send-mail@v2 From 52e14e528e751bacb01d1f00b908b015cca0ce7d Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Sat, 29 Oct 2022 09:21:03 -0400 Subject: [PATCH 08/36] Cleanup. --- core/unix/os.c | 6 +++--- core/unix/rseq_linux.c | 27 ++++++++++++++++++--------- core/unix/rseq_linux.h | 2 +- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/core/unix/os.c b/core/unix/os.c index 161749af328..ec440543105 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -9363,7 +9363,7 @@ post_system_call(dcontext_t *dcontext) /* Lazy rseq handling. */ if (success) { rseq_process_syscall(dcontext); - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(true); } break; #endif @@ -10729,7 +10729,7 @@ os_take_over_all_unknown_threads(dcontext_t *dcontext) #ifdef LINUX /* Check this thread for rseq in between setup and start. */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); #endif /* Find tids for which we have no thread record, meaning they are not under @@ -10957,7 +10957,7 @@ os_thread_take_over(priv_mcontext_t *mc, kernel_sigset_t *sigset) * regions as rseq when the rseq syscall is never set up. */ if (rseq_is_registered_for_current_thread()) { - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); rseq_thread_attach(dcontext); } #endif diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index acf42777674..b821b79ab7f 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -50,6 +50,7 @@ #include #include "include/syscall.h" #include + /* The linux/rseq.h header made a source-breaking change in torvalds/linux@bfdf4e6 * which broke our build. To avoid future issues we use our own definitions. * Binary breakage is unlikely without long periods of deprecation so this is @@ -175,7 +176,9 @@ rseq_check_glibc_enabled() /* __rseq_size is set to zero if the rseq support in glibc is disabled * by exporting GLIBC_TUNABLES=glibc.pthread.rseq=0. Or it's possible * that we're using early inject and it just hasn't been initialized - * yet. + * yet. We prevent the latter from happening by calling this routine + * after modules are initialised (on static DR), and after we reach + * the image entry (set_reached_image_entry; on non-static DR). */ if (rseq_size_addr == NULL || *rseq_size_addr == 0) continue; @@ -188,18 +191,15 @@ rseq_check_glibc_enabled() ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); DODEBUG({ byte *rseq_addr = get_app_segment_base(LIB_SEG_TLS) + rseq_tls_offset; - LOG(GLOBAL, LOG_LOADER, 2, - "Found struct rseq registered by glibc @ " PFX " at TLS offset %d\n", - rseq_addr, rseq_tls_offset); int res = dynamorio_syscall(SYS_rseq, 4, rseq_addr, sizeof(struct rseq), 0, 0); ASSERT(res == -EPERM); + LOG(GLOBAL, LOG_LOADER, 2, + "Found struct rseq registered by glibc @ " PFX " at TLS offset %d\n", + rseq_addr, rseq_tls_offset); }); } else { ASSERT(rseq_tls_offset == *rseq_offset_addr); -# ifdef STATIC_LIBRARY - ASSERT(glibc_rseq_enabled); -# endif } } module_iterator_stop(iter); @@ -220,7 +220,7 @@ d_r_rseq_init(void) rseq_check_glibc_enabled(); /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); #else /* For non-static DR (early-inject), we delay enabling rseq till libc is * completely initialized, in set_reached_image_entry. @@ -234,6 +234,7 @@ d_r_rseq_exit(void) generic_hash_destroy(GLOBAL_DCONTEXT, rseq_cs_table); vmvector_delete_vector(GLOBAL_DCONTEXT, d_r_rseq_areas); DELETE_LOCK(rseq_trigger_lock); + /* Better to reset and detect again for reattaches. */ #ifdef GLIBC_RSEQ bool new_value = false; ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); @@ -801,7 +802,7 @@ rseq_process_syscall(dcontext_t *dcontext) * the app has registered the current thread for rseq. */ void -rseq_locate_rseq_regions(void) +rseq_locate_rseq_regions(bool at_syscall) { if (rseq_enabled) return; @@ -810,6 +811,14 @@ rseq_locate_rseq_regions(void) * completely initialized. */ if (!reached_image_entry_yet()) { + /* This is likely the initial rseq made by glibc. Since glibc's + * rseq support isn't fully initialized yet (the __rseq_size and + * __rseq_offset values; assuming glibc rseq support isn't disabled + * explicitly by the user), we skip locating rseq regions for now. + * We'll do it when the thread reaches the image entry + * (in set_reached_image_entry). + */ + ASSERT(at_syscall); return; } #endif diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index d33ef70dda8..9f4084cc373 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -46,7 +46,7 @@ #endif #if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 35) -# define GLIBC_RSEQ +# define GLIBC_RSEQ 1 #endif void From 64cb13188abe09eec52e238b96c25a9edd923f52 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Sat, 29 Oct 2022 09:21:29 -0400 Subject: [PATCH 09/36] Revert "Revert "Add tmate"" This reverts commit 0a0dfce465d2ae59988fdb1005146bb9d83a0321. --- .github/workflows/ci-x86.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci-x86.yml b/.github/workflows/ci-x86.yml index f2b4da11f8b..11416c64f50 100644 --- a/.github/workflows/ci-x86.yml +++ b/.github/workflows/ci-x86.yml @@ -172,6 +172,13 @@ jobs: CI_TRIGGER: ${{ github.event_name }} CI_BRANCH: ${{ github.ref }} + - name: Setup tmate session on failure + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + + - name: Setup tmate session on success + uses: mxschmitt/action-tmate@v3 + - name: Send failure mail to dynamorio-devs if: failure() && github.ref == 'refs/heads/master' uses: dawidd6/action-send-mail@v2 From cd069e5bb5502e93d6dca08ff5f0d610df3b4883 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Sat, 29 Oct 2022 09:24:31 -0400 Subject: [PATCH 10/36] Fix build --- core/arch/interp.c | 2 +- core/unix/rseq_linux.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/arch/interp.c b/core/arch/interp.c index 7667a772cb2..421af7c04ba 100644 --- a/core/arch/interp.c +++ b/core/arch/interp.c @@ -383,7 +383,7 @@ set_reached_image_entry() */ rseq_check_glibc_enabled(); if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); #endif } diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index 9f4084cc373..cb7d8d0e507 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -62,7 +62,7 @@ bool rseq_is_registered_for_current_thread(void); void -rseq_locate_rseq_regions(void); +rseq_locate_rseq_regions(bool); void rseq_module_init(module_area_t *ma, bool at_map); From 51190be0c8567c2af9ded3fa54babf5408f788b8 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Mon, 31 Oct 2022 15:59:47 -0400 Subject: [PATCH 11/36] Use dynamo_control_via_attach to detect attach mode --- core/arch/interp.c | 20 +++++++++++--------- core/unix/os.c | 2 +- core/unix/rseq_linux.c | 36 +++++++++++++++++------------------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/core/arch/interp.c b/core/arch/interp.c index 421af7c04ba..efecdebf9ef 100644 --- a/core/arch/interp.c +++ b/core/arch/interp.c @@ -66,7 +66,7 @@ #include "../native_exec.h" #include "../jit_opt.h" -#if !defined(STATIC_LIBRARY) && defined(LINUX) +#if defined(LINUX) # include "../unix/rseq_linux.h" #endif #ifdef CHECK_RETURNS_SSE2 @@ -376,14 +376,16 @@ set_reached_image_entry() SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); reached_image_entry = true; SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); -#if !defined(STATIC_LIBRARY) && defined(LINUX) - /* For enabling rseq on non-static DR, we need to wait till libc is - * initialized. This is a good control point. - * Note that d_r_rseq_init would already have completed by now. - */ - rseq_check_glibc_enabled(); - if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(false); +#if defined(LINUX) + if (!dynamo_control_via_attach) { + /* For enabling rseq in the non-attach case, we need to wait till libc is + * initialized. This is a good control point. + * Note that d_r_rseq_init would already have completed by now. + */ + rseq_check_glibc_enabled(); + if (rseq_is_registered_for_current_thread()) + rseq_locate_rseq_regions(false); + } #endif } diff --git a/core/unix/os.c b/core/unix/os.c index ec440543105..211efa294c6 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -10067,7 +10067,7 @@ find_executable_vm_areas(void) /* now that we have the modules set up, query libc */ get_libc_errno_location(true /*force init*/); -#if !defined(STATIC_LIBRARY) && defined(LINUX) +#if defined(LINUX) /* For rseq on GLIBC >= 2.35, we need to find __rseq_size and __rseq_offset, * but we cannot do that here as they wouldn't be initialized yet. */ diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index b821b79ab7f..2652ab04b41 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -160,14 +160,12 @@ rseq_check_glibc_enabled() { #ifdef GLIBC_RSEQ ASSERT(!glibc_rseq_enabled); -# ifdef STATIC_LIBRARY - ASSERT(rseq_tls_offset == 0); -# else - /* For !STATIC_LIBRARY we delay rseq_check_glibc_enabled until we GLIBC is - * initialized, so it may happen that rseq_tls_offset is already initialized - * by rseq_process_syscall. + /* For !dynamo_control_via_attach, we delay rseq_check_glibc_enabled until + * GLIBC is initialized, so it may happen that rseq_tls_offset is already + * initialized by rseq_process_syscall. */ -# endif + ASSERT(!dynamo_control_via_attach || rseq_tls_offset == 0); + module_iterator_t *iter = module_iterator_start(); while (module_iterator_hasnext(iter)) { module_area_t *ma = module_iterator_next(iter); @@ -216,16 +214,16 @@ d_r_rseq_init(void) rseq_cs_table = generic_hash_create(GLOBAL_DCONTEXT, INIT_RSEQ_CS_TABLE_SIZE, 80, HASHTABLE_SHARED | HASHTABLE_PERSISTENT, rseq_cs_free _IF_DEBUG("rseq_cs table")); -#ifdef STATIC_LIBRARY - rseq_check_glibc_enabled(); - /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ - if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(false); -#else - /* For non-static DR (early-inject), we delay enabling rseq till libc is + + /* For the non-attach case, we delay enabling rseq till libc is * completely initialized, in set_reached_image_entry. */ -#endif + if (dynamo_control_via_attach) { + rseq_check_glibc_enabled(); + /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ + if (rseq_is_registered_for_current_thread()) + rseq_locate_rseq_regions(false); + } } void @@ -806,11 +804,11 @@ rseq_locate_rseq_regions(bool at_syscall) { if (rseq_enabled) return; -#if defined(GLIBC_RSEQ) && !defined(STATIC_LIBRARY) - /* We delay locating struct rseq TLS offset and rseq_cs until GLIBC is - * completely initialized. +#if defined(GLIBC_RSEQ) + /* For the non-attach case, we delay locating struct rseq TLS offset and + * rseq_cs until GLIBC is completely initialized. */ - if (!reached_image_entry_yet()) { + if (!dynamo_control_via_attach && !reached_image_entry_yet()) { /* This is likely the initial rseq made by glibc. Since glibc's * rseq support isn't fully initialized yet (the __rseq_size and * __rseq_offset values; assuming glibc rseq support isn't disabled From ba77f107f9f168336d035bc829c45995601154c8 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Mon, 31 Oct 2022 17:30:03 -0400 Subject: [PATCH 12/36] Revert ubuntu-22 test filters. --- suite/runsuite.cmake | 13 ------------- suite/tests/CMakeLists.txt | 18 ------------------ 2 files changed, 31 deletions(-) diff --git a/suite/runsuite.cmake b/suite/runsuite.cmake index 1e6040628bf..2d02c2a945e 100644 --- a/suite/runsuite.cmake +++ b/suite/runsuite.cmake @@ -77,19 +77,6 @@ foreach (arg ${CTEST_SCRIPT_ARG}) endif () endforeach (arg) -if (UNIX AND NOT APPLE AND NOT ANDROID) - execute_process(COMMAND ldd --version - RESULT_VARIABLE ldd_result ERROR_VARIABLE ldd_err OUTPUT_VARIABLE ldd_out) - if (ldd_result OR ldd_err) - # Failed; just move on. - elseif (ldd_out MATCHES "GLIBC 2.3[5-9]") - # XXX i#5437, i#5431: While we work through Ubuntu22 issues we run - # just a few tests. - set(extra_ctest_args INCLUDE_LABEL UBUNTU_22) - set(arg_debug_only ON) - endif () -endif () - set(build_tests "BUILD_TESTS:BOOL=ON") if (arg_automated_ci) diff --git a/suite/tests/CMakeLists.txt b/suite/tests/CMakeLists.txt index b0e8c249ad5..a7309c33d77 100644 --- a/suite/tests/CMakeLists.txt +++ b/suite/tests/CMakeLists.txt @@ -5355,21 +5355,3 @@ if (NOT ANDROID AND AARCHXX) PROPERTIES LABELS RUNS_ON_QEMU) endif () endif (NOT ANDROID AND AARCHXX) - -# XXX i#5437, i#5431: While we work through Ubuntu22 issues we want our new GA CI job to -# just run sanity tests that do pass now as a glibc change detector to see if our -# workarounds so far break over time. -set_tests_properties( - code_api|common.broadfun - PROPERTIES LABELS UBUNTU_22) -if (BUILD_SAMPLES AND NOT ANDROID) - set_tests_properties( - code_api|sample.bbsize - PROPERTIES LABELS UBUNTU_22) -endif () -if (BUILD_CLIENTS AND NOT ANDROID) - set_tests_properties( - code_api|tool.drcachesim.simple - code_api|tool.drcacheoff.simple - PROPERTIES LABELS UBUNTU_22) -endif () From 79d80f34fedcdff3fe5bc7fbeb6e5c15f1f7156d Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Mon, 31 Oct 2022 17:31:25 -0400 Subject: [PATCH 13/36] Remove the reached image entry hook, which is not very reliable. --- core/ir/riscv64/instr_create_api.h | 373 +++++++++++++++++++++++++++++ core/ir/riscv64/opcode_api.h | 92 +++++++ core/unix/os.c | 2 +- core/unix/rseq_linux.c | 119 +++++---- core/unix/rseq_linux.h | 2 +- 5 files changed, 521 insertions(+), 67 deletions(-) create mode 100644 core/ir/riscv64/instr_create_api.h create mode 100644 core/ir/riscv64/opcode_api.h diff --git a/core/ir/riscv64/instr_create_api.h b/core/ir/riscv64/instr_create_api.h new file mode 100644 index 00000000000..f29bae65a73 --- /dev/null +++ b/core/ir/riscv64/instr_create_api.h @@ -0,0 +1,373 @@ +/* ********************************************************** + * Copyright (c) 2022 Rivos, Inc. All rights reserved. + * **********************************************************/ + +/* + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of Rivos, Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL RIVOS, INC. OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + */ + +#ifndef _DR_IR_MACROS_RISCV64_H_ +#define _DR_IR_MACROS_RISCV64_H_ 1 + +/**************************************************************************** + * Platform-independent INSTR_CREATE_* macros + */ +/** @name Platform-independent macros */ +/** @{ */ /* doxygen start group */ + +/* FIXME i#3544: Check and implement all INSTR_CREATE_ and XINST_CREATE_ macros. For now + * all unimplemented instructions use OP_hint which is effectively a no-op. */ + +/** + * This platform-independent macro creates an instr_t for a debug trap + * instruction, automatically supplying any implicit operands. + * \param dc The void * dcontext used to allocate memory for the instr_t. + */ +#define XINST_CREATE_debug_instr(dc) instr_create_0dst_0src(dc, OP_hint) + +/** + * This platform-independent macro creates an instr_t for a 4-byte + * or 8-byte memory load instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param r The destination register opnd. + * \param m The source memory opnd. + */ +#define XINST_CREATE_load(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) + +/** + * This platform-independent macro creates an instr_t which loads 1 byte + * from memory, zero-extends it to 4 bytes, and writes it to a 4 byte + * destination register. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param r The destination register opnd. + * \param m The source memory opnd. + */ +#define XINST_CREATE_load_1byte_zext4(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) + +/** + * This platform-independent macro creates an instr_t for a 1-byte + * memory load instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param r The destination register opnd. + * \param m The source memory opnd. + */ +#define XINST_CREATE_load_1byte(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) + +/** + * This platform-independent macro creates an instr_t for a 2-byte + * memory load instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param r The destination register opnd. + * \param m The source memory opnd. + */ +#define XINST_CREATE_load_2bytes(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) + +/** + * This platform-independent macro creates an instr_t for a 4-byte + * or 8-byte memory store instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param m The destination memory opnd. + * \param r The source register opnd. + */ +#define XINST_CREATE_store(dc, m, r) instr_create_1dst_1src(dc, OP_hint, m, r) + +/** + * This platform-independent macro creates an instr_t for a 1-byte + * memory store instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param m The destination memory opnd. + * \param r The source register opnd. + */ +#define XINST_CREATE_store_1byte(dc, m, r) instr_create_1dst_1src(dc, OP_hint, m, r) + +/** + * This platform-independent macro creates an instr_t for a 2-byte + * memory store instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param m The destination memory opnd. + * \param r The source register opnd. + */ +#define XINST_CREATE_store_2bytes(dc, m, r) instr_create_1dst_1src(dc, OP_hint, m, r) + +/** + * This platform-independent macro creates an instr_t for a register + * to register move instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The destination register opnd. + * \param s The source register opnd. + */ +#define XINST_CREATE_move(dc, d, s) instr_create_1dst_1src(dc, OP_hint, d, s) + +/** + * This platform-independent macro creates an instr_t for a multimedia + * register load instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param r The destination register opnd. + * \param m The source memory opnd. + */ +#define XINST_CREATE_load_simd(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) + +/** + * This platform-independent macro creates an instr_t for a multimedia + * register store instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param m The destination memory opnd. + * \param r The source register opnd. + */ +#define XINST_CREATE_store_simd(dc, m, r) instr_create_1dst_1src(dc, OP_hint, m, r) + +/** + * This platform-independent macro creates an instr_t for an indirect + * jump through memory instruction. For AArch32, the memory address + * must be aligned to 4. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param m The memory opnd holding the target. + */ +#define XINST_CREATE_jump_mem(dc, m) \ + instr_create_0dst_2src(dc, OP_hint, opnd_create_reg(DR_REG_PC), (m)) + +/** + * This platform-independent macro creates an instr_t for an indirect + * jump instruction through a register. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param r The register opnd holding the target. + */ +#define XINST_CREATE_jump_reg(dc, r) instr_create_0dst_1src(dc, OP_hint, r) + +/** + * This platform-independent macro creates an instr_t for an immediate + * integer load instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param r The destination register opnd. + * \param i The source immediate integer opnd. + */ +#define XINST_CREATE_load_int(dc, r, i) instr_create_1dst_1src(dc, OP_hint, r, i) + +/** + * This platform-independent macro creates an instr_t for a return instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + */ +#define XINST_CREATE_return(dc) instr_create_0dst_0src(dc, OP_hint) + +/** + * This platform-independent macro creates an instr_t for an unconditional + * branch instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param t The opnd_t target operand for the instruction, which can be + * either a pc (opnd_create_pc)()) or an instr_t (opnd_create_instr()). + * Be sure to ensure that the limited reach of this short branch will reach + * the target (a pc operand is not suitable for most uses unless you know + * precisely where this instruction will be encoded). + */ +#define XINST_CREATE_jump(dc, t) instr_create_0dst_1src(dc, OP_hint, t) + +/** + * This platform-independent macro creates an instr_t for an unconditional + * branch instruction with the smallest available reach. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param t The opnd_t target operand for the instruction, which can be + * either a pc (opnd_create_pc)()) or an instr_t (opnd_create_instr()). + * Be sure to ensure that the limited reach of this short branch will reach + * the target (a pc operand is not suitable for most uses unless you know + * precisely where this instruction will be encoded). + */ +#define XINST_CREATE_jump_short(dc, t) instr_create_0dst_1src(dc, OP_hint, t) + +/** + * This platform-independent macro creates an instr_t for an unconditional + * branch instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param t The opnd_t target operand for the instruction, which can be + * either a pc (opnd_create_pc)()) or an instr_t (opnd_create_instr()). + * Be sure to ensure that the limited reach of this short branch will reach + * the target (a pc operand is not suitable for most uses unless you know + * precisely where this instruction will be encoded). + */ +#define XINST_CREATE_call(dc, t) instr_create_0dst_1src(dc, OP_hint, t) + +/** + * This platform-independent macro creates an instr_t for a conditional + * branch instruction that branches if the previously-set condition codes + * indicate the condition indicated by \p pred. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param pred The #dr_pred_type_t condition to match. + * \param t The opnd_t target operand for the instruction, which can be + * either a pc (opnd_create_pc)()) or an instr_t (opnd_create_instr()). + * Be sure to ensure that the limited reach of this short branch will reach + * the target (a pc operand is not suitable for most uses unless you know + * precisely where this instruction will be encoded). + */ +#define XINST_CREATE_jump_cond(dc, pred, t) \ + (INSTR_PRED(instr_create_0dst_1src(dc, OP_hint, t), (pred))) + +/** + * This platform-independent macro creates an instr_t for an addition + * instruction that does not affect the status flags. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The opnd_t explicit destination operand for the instruction. + * \param s The opnd_t explicit source operand for the instruction. + */ +#define XINST_CREATE_add(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) + +/** + * This platform-independent macro creates an instr_t for an addition + * instruction that does not affect the status flags and takes two sources + * plus a destination. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The opnd_t explicit destination operand for the instruction. + * \param s1 The opnd_t explicit first source operand for the instruction. This + * must be a register. + * \param s2 The opnd_t explicit source operand for the instruction. This + * can be either a register or an immediate integer. + */ +#define XINST_CREATE_add_2src(dc, d, s1, s2) \ + instr_create_1dst_2src(dc, OP_hint, d, s1, s2) + +/** + * This platform-independent macro creates an instr_t for an addition + * instruction that does not affect the status flags and takes two register sources + * plus a destination, with one source being shifted logically left by + * an immediate amount that is limited to either 0, 1, 2, or 3. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The opnd_t explicit destination operand for the instruction. + * \param s1 The opnd_t explicit first source operand for the instruction. This + * must be a register. + * \param s2_toshift The opnd_t explicit source operand for the instruction. This + * must be a register. + * \param shift_amount An integer value that must be either 0, 1, 2, or 3. + */ +#define XINST_CREATE_add_sll(dc, d, s1, s2_toshift, shift_amount) \ + instr_create_1dst_3src(dc, OP_hint, d, s1, s2_toshift, \ + opnd_create_immed_int(shift_amount, OPSZ_1)) + +/** + * This platform-independent macro creates an instr_t for an addition + * instruction that does affect the status flags. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The opnd_t explicit destination operand for the instruction. + * \param s The opnd_t explicit source operand for the instruction. + */ +#define XINST_CREATE_add_s(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) + +/** + * This platform-independent macro creates an instr_t for a subtraction + * instruction that does not affect the status flags. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The opnd_t explicit destination operand for the instruction. + * \param s The opnd_t explicit source operand for the instruction. + */ +/* FIXME: How to handle a lack of flags register? */ +#define XINST_CREATE_sub(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) + +/** + * This platform-independent macro creates an instr_t for a subtraction + * instruction that does affect the status flags. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The opnd_t explicit destination operand for the instruction. + * \param s The opnd_t explicit source operand for the instruction. + */ +/* FIXME: How to handle a lack of flags register? */ +#define XINST_CREATE_sub_s(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) + +/** + * This platform-independent macro creates an instr_t for a bitwise and + * instruction that does affect the status flags. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The opnd_t explicit destination operand for the instruction. + * \param s The opnd_t explicit source operand for the instruction. + */ +/* FIXME: How to handle a lack of flags register? */ +#define XINST_CREATE_and_s(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) + +/** + * This platform-independent macro creates an instr_t for a comparison + * instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param s1 The opnd_t explicit source operand for the instruction. + * \param s2 The opnd_t explicit source operand for the instruction. + */ +/* FIXME: How to handle a lack of flags register? */ +#define XINST_CREATE_cmp(dc, s1, s2) instr_create_0dst_2src(dc, OP_hint, s1, s2) + +/** + * This platform-independent macro creates an instr_t for a software + * interrupt instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param i The source integer constant opnd_t operand. + */ +#define XINST_CREATE_interrupt(dc, i) instr_create_0dst_1src(dc, OP_hint, i) + +/** + * This platform-independent macro creates an instr_t for a logical right shift + * instruction that does affect the status flags. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param d The opnd_t explicit destination operand for the instruction. + * \param rm_or_imm The opnd_t explicit source operand for the instruction. + */ +#define XINST_CREATE_slr_s(dc, d, rm_or_imm) \ + instr_create_1dst_1src(dc, OP_hint, d, rm_or_imm) + +/** + * This platform-independent macro creates an instr_t for a nop instruction. + * \param dc The void * dcontext used to allocate memory for the instr_t. + */ +#define XINST_CREATE_nop(dc) INSTR_CREATE_nop(dc) + +/** + * This platform-independent macro creates an instr_t for an indirect call instr + * through a register. + * \param dc The void * dcontext used to allocate memory for the instr_t. + * \param r The opnd_t explicit source operand for the instruction. This should + * be a reg_id_t operand with the address of the subroutine. + */ +#define XINST_CREATE_call_reg(dc, r) instr_create_0dst_1src(dc, OP_hint, r) + +/** + * Create an absolute address operand encoded as pc-relative. + * Encoding will fail if addr is out of the maximum signed displacement + * reach for the architecture. + */ +#define OPND_CREATE_ABSMEM(addr, size) opnd_create_rel_addr(addr, size) + +/** @} */ /* end doxygen group */ + +/**************************************************************************** + * RISC-V-specific INSTR_CREATE_* macros + */ + +/** @name No-operand instructions */ +/** @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */ +/** + * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically + * supplying any implicit operands. + * \param dc The void * dcontext used to allocate memory for the instr_t. + */ +#define INSTR_CREATE_ecall(dc) instr_create_0dst_0src(dc, OP_ecall) +#define INSTR_CREATE_nop(dc) instr_create_0dst_0src(dc, OP_hint) +/** @} */ /* end doxygen group */ + +#endif /* _DR_IR_MACROS_RISCV64_H_ */ diff --git a/core/ir/riscv64/opcode_api.h b/core/ir/riscv64/opcode_api.h new file mode 100644 index 00000000000..dbc7e05f212 --- /dev/null +++ b/core/ir/riscv64/opcode_api.h @@ -0,0 +1,92 @@ +/* ********************************************************** + * Copyright (c) 2022 Rivos, Inc. All rights reserved. + * **********************************************************/ + +/* + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * * Neither the name of Rivos, Inc. nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL RIVOS, INC. OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + */ + +#ifndef _DR_IR_OPCODES_RISCV64_H_ +#define _DR_IR_OPCODES_RISCV64_H_ 1 + +/**************************************************************************** + * OPCODES + */ +/** + * @file dr_ir_opcodes_arm.h + * @brief Instruction opcode constants for RISC-V. + */ +/** Opcode constants for use in the instr_t data structure. */ +enum { + /* 0 */ OP_INVALID, + /* NULL, */ /**< INVALID opcode */ + /* 1 */ OP_UNDECODED, + /* NULL, */ /**< UNDECODED opcode */ + /* 2 */ OP_CONTD, + /* NULL, */ /**< CONTD opcode */ + /* 3 */ OP_LABEL, + /* NULL, */ /**< LABEL opcode */ + + /* FIXME i#3544: Put real opcodes here and make sure ordering matches HW. */ + OP_ecall, /**< RISC-V ecall opcode. */ + OP_l, /**< RISC-V integer load opcode. Width is encoded in operand sizes. */ + OP_s, /**< RISC-V integer store opcode. Width is encoded in operand sizes. */ + OP_add, /**< RISC-V add opcode. */ + OP_addi, /**< RISC-V addi opcode. */ + OP_sub, /**< RISC-V sub opcode. */ + OP_mul, /**< RISC-V mul opcode. */ + OP_mulu, /**< RISC-V mulu opcode. */ + OP_div, /**< RISC-V div opcode. */ + OP_divu, /**< RISC-V divu opcode. */ + OP_jal, /**< RISC-V jal opcode. */ + OP_jalr, /**< RISC-V jalr opcode. */ + OP_beq, /**< RISC-V beq opcode. */ + OP_bne, /**< RISC-V bne opcode. */ + OP_blt, /**< RISC-V blt opcode. */ + OP_bltu, /**< RISC-V bltu opcode. */ + OP_bge, /**< RISC-V bge opcode. */ + OP_bgeu, /**< RISC-V bgeu opcode. */ + OP_hint, /**< RISC-V hint opcode. */ + +#if defined(RISCV_ISA_F) || defined(RISCV_ISA_D) + OP_fl, /**< RISC-V FP load opcode. Width is encoded in operand sizes. */ + OP_fs, /**< RISC-V FP store opcode. Width is encoded in operand sizes. */ +#endif + + OP_AFTER_LAST, + /* FIXME i#3544: Replace OP_hint with a real opcode. */ + OP_FIRST = OP_hint, /**< First real opcode. */ + OP_LAST = OP_AFTER_LAST - 1, /**< Last real opcode. */ +}; + +/* alternative names */ +#define OP_jmp OP_hint /**< Platform-independent opcode name for jump. */ +#define OP_jmp_short OP_hint /**< Platform-independent opcode name for short jump. */ +#define OP_load OP_hint /**< Platform-independent opcode name for load. */ +#define OP_store OP_hint /**< Platform-independent opcode name for store. */ + +#endif /* _DR_IR_OPCODES_RISCV64_H_ */ diff --git a/core/unix/os.c b/core/unix/os.c index 211efa294c6..a73d0979b5e 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -9363,7 +9363,7 @@ post_system_call(dcontext_t *dcontext) /* Lazy rseq handling. */ if (success) { rseq_process_syscall(dcontext); - rseq_locate_rseq_regions(true); + rseq_locate_rseq_regions(); } break; #endif diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 2652ab04b41..52c9b49c3a9 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -74,7 +74,13 @@ struct rseq { #define RSEQ_FLAG_UNREGISTER 1 #ifdef GLIBC_RSEQ -static volatile bool glibc_rseq_enabled; +/* __tls_init_tp in glibc/sysdeps/nptl/dl-tls_init_tp.c initializes __rseq_size + * and __rseq_offset. The former is > 0 iff glibc's rseq support is not disabled + * using GLIBC_TUNABLES=glibc.pthread.rseq=0. The latter is defined regardless + * and is used by us to determine whether glibc has been initialized yet. + */ +static volatile uint glibc_rseq_size; +static volatile int glibc_rseq_offset; #endif vm_area_vector_t *d_r_rseq_areas; @@ -159,46 +165,39 @@ void rseq_check_glibc_enabled() { #ifdef GLIBC_RSEQ - ASSERT(!glibc_rseq_enabled); - /* For !dynamo_control_via_attach, we delay rseq_check_glibc_enabled until - * GLIBC is initialized, so it may happen that rseq_tls_offset is already - * initialized by rseq_process_syscall. - */ - ASSERT(!dynamo_control_via_attach || rseq_tls_offset == 0); - + /* We have already determined Glibc's rseq support. */ + if (glibc_rseq_offset > 0) + return; module_iterator_t *iter = module_iterator_start(); while (module_iterator_hasnext(iter)) { module_area_t *ma = module_iterator_next(iter); + int *rseq_offset_addr = + (int *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_offset"); + /* Glibc's not initialized yet. */ + if (rseq_offset_addr == NULL || *rseq_offset_addr == 0) + continue; + /* Verify that the offset has not changed. */ + ASSERT(glibc_rseq_offset == 0 || + (glibc_rseq_offset == *rseq_offset_addr && + rseq_tls_offset == *rseq_offset_addr)); + ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, *rseq_offset_addr, false); + ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); uint *rseq_size_addr = (uint *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_size"); - /* __rseq_size is set to zero if the rseq support in glibc is disabled - * by exporting GLIBC_TUNABLES=glibc.pthread.rseq=0. Or it's possible - * that we're using early inject and it just hasn't been initialized - * yet. We prevent the latter from happening by calling this routine - * after modules are initialised (on static DR), and after we reach - * the image entry (set_reached_image_entry; on non-static DR). - */ - if (rseq_size_addr == NULL || *rseq_size_addr == 0) + ASSERT(rseq_size_addr != NULL); + /* Glibc's rseq support is disabled. */ + if (*rseq_size_addr == 0) continue; - int *rseq_offset_addr = - (int *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_offset"); - ASSERT(rseq_offset_addr != NULL); - if (rseq_tls_offset == 0) { - ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); - bool new_value = true; - ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); - DODEBUG({ - byte *rseq_addr = get_app_segment_base(LIB_SEG_TLS) + rseq_tls_offset; - int res = - dynamorio_syscall(SYS_rseq, 4, rseq_addr, sizeof(struct rseq), 0, 0); - ASSERT(res == -EPERM); - LOG(GLOBAL, LOG_LOADER, 2, - "Found struct rseq registered by glibc @ " PFX " at TLS offset %d\n", - rseq_addr, rseq_tls_offset); - }); - } else { - ASSERT(rseq_tls_offset == *rseq_offset_addr); - } + ATOMIC_4BYTE_WRITE(&glibc_rseq_size, *rseq_size_addr, false); + DODEBUG({ + byte *rseq_addr = get_app_segment_base(LIB_SEG_TLS) + glibc_rseq_offset; + int res = + dynamorio_syscall(SYS_rseq, 4, rseq_addr, sizeof(struct rseq), 0, 0); + ASSERT(res == -EPERM); + LOG(GLOBAL, LOG_LOADER, 2, + "Found struct rseq registered by glibc @ " PFX " at TLS offset %d\n", + rseq_addr, rseq_tls_offset); + }); } module_iterator_stop(iter); #endif @@ -214,16 +213,10 @@ d_r_rseq_init(void) rseq_cs_table = generic_hash_create(GLOBAL_DCONTEXT, INIT_RSEQ_CS_TABLE_SIZE, 80, HASHTABLE_SHARED | HASHTABLE_PERSISTENT, rseq_cs_free _IF_DEBUG("rseq_cs table")); - - /* For the non-attach case, we delay enabling rseq till libc is - * completely initialized, in set_reached_image_entry. - */ - if (dynamo_control_via_attach) { - rseq_check_glibc_enabled(); - /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ - if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(false); - } + rseq_check_glibc_enabled(); + /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ + if (rseq_is_registered_for_current_thread()) + rseq_locate_rseq_regions(); } void @@ -233,12 +226,12 @@ d_r_rseq_exit(void) vmvector_delete_vector(GLOBAL_DCONTEXT, d_r_rseq_areas); DELETE_LOCK(rseq_trigger_lock); /* Better to reset and detect again for reattaches. */ + uint new_value = 0; + ATOMIC_4BYTE_WRITE(&rseq_tls_offset, new_value, false); #ifdef GLIBC_RSEQ - bool new_value = false; - ATOMIC_1BYTE_WRITE(&glibc_rseq_enabled, new_value, false); + ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, new_value, false); + ATOMIC_4BYTE_WRITE(&glibc_rseq_size, new_value, false); #endif - uint new_offset = 0; - ATOMIC_4BYTE_WRITE(&rseq_tls_offset, new_offset, false); } void @@ -378,7 +371,10 @@ bool rseq_is_registered_for_current_thread(void) { #ifdef GLIBC_RSEQ - if (glibc_rseq_enabled) + /* If Glibc's rseq support is enabled, the struct rseq is already + * registered for all threads. + */ + if (glibc_rseq_size > 0) return true; #endif /* Unfortunately there's no way to query the current rseq struct. @@ -699,8 +695,9 @@ static int rseq_locate_tls_offset(void) { #ifdef GLIBC_RSEQ - if (glibc_rseq_enabled) { - ASSERT(rseq_tls_offset != 0); + /* If Glibc's rseq support is enabled, we already know the offset. */ + if (glibc_rseq_size > 0) { + ASSERT(rseq_tls_offset > 0); return rseq_tls_offset; } #endif @@ -800,25 +797,17 @@ rseq_process_syscall(dcontext_t *dcontext) * the app has registered the current thread for rseq. */ void -rseq_locate_rseq_regions(bool at_syscall) +rseq_locate_rseq_regions() { if (rseq_enabled) return; #if defined(GLIBC_RSEQ) - /* For the non-attach case, we delay locating struct rseq TLS offset and - * rseq_cs until GLIBC is completely initialized. + rseq_check_glibc_enabled(); + /* Glibc's rseq support has not been initialized yet. Return for now and + * try again later. */ - if (!dynamo_control_via_attach && !reached_image_entry_yet()) { - /* This is likely the initial rseq made by glibc. Since glibc's - * rseq support isn't fully initialized yet (the __rseq_size and - * __rseq_offset values; assuming glibc rseq support isn't disabled - * explicitly by the user), we skip locating rseq regions for now. - * We'll do it when the thread reaches the image entry - * (in set_reached_image_entry). - */ - ASSERT(at_syscall); + if (glibc_rseq_offset == 0) return; - } #endif /* This is a global operation, but the trigger could be hit by two threads at once, * thus requiring synchronization. diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index cb7d8d0e507..71a61f7bf99 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -62,7 +62,7 @@ bool rseq_is_registered_for_current_thread(void); void -rseq_locate_rseq_regions(bool); +rseq_locate_rseq_regions(); void rseq_module_init(module_area_t *ma, bool at_map); From b80ec25db039f0e558bd074f48779a57b42ec8bd Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Mon, 31 Oct 2022 17:33:12 -0400 Subject: [PATCH 14/36] Remove spurious files. --- core/ir/riscv64/instr_create_api.h | 373 ----------------------------- core/ir/riscv64/opcode_api.h | 92 ------- 2 files changed, 465 deletions(-) delete mode 100644 core/ir/riscv64/instr_create_api.h delete mode 100644 core/ir/riscv64/opcode_api.h diff --git a/core/ir/riscv64/instr_create_api.h b/core/ir/riscv64/instr_create_api.h deleted file mode 100644 index f29bae65a73..00000000000 --- a/core/ir/riscv64/instr_create_api.h +++ /dev/null @@ -1,373 +0,0 @@ -/* ********************************************************** - * Copyright (c) 2022 Rivos, Inc. All rights reserved. - * **********************************************************/ - -/* - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * * Neither the name of Rivos, Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL RIVOS, INC. OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - * DAMAGE. - */ - -#ifndef _DR_IR_MACROS_RISCV64_H_ -#define _DR_IR_MACROS_RISCV64_H_ 1 - -/**************************************************************************** - * Platform-independent INSTR_CREATE_* macros - */ -/** @name Platform-independent macros */ -/** @{ */ /* doxygen start group */ - -/* FIXME i#3544: Check and implement all INSTR_CREATE_ and XINST_CREATE_ macros. For now - * all unimplemented instructions use OP_hint which is effectively a no-op. */ - -/** - * This platform-independent macro creates an instr_t for a debug trap - * instruction, automatically supplying any implicit operands. - * \param dc The void * dcontext used to allocate memory for the instr_t. - */ -#define XINST_CREATE_debug_instr(dc) instr_create_0dst_0src(dc, OP_hint) - -/** - * This platform-independent macro creates an instr_t for a 4-byte - * or 8-byte memory load instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param r The destination register opnd. - * \param m The source memory opnd. - */ -#define XINST_CREATE_load(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) - -/** - * This platform-independent macro creates an instr_t which loads 1 byte - * from memory, zero-extends it to 4 bytes, and writes it to a 4 byte - * destination register. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param r The destination register opnd. - * \param m The source memory opnd. - */ -#define XINST_CREATE_load_1byte_zext4(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) - -/** - * This platform-independent macro creates an instr_t for a 1-byte - * memory load instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param r The destination register opnd. - * \param m The source memory opnd. - */ -#define XINST_CREATE_load_1byte(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) - -/** - * This platform-independent macro creates an instr_t for a 2-byte - * memory load instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param r The destination register opnd. - * \param m The source memory opnd. - */ -#define XINST_CREATE_load_2bytes(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) - -/** - * This platform-independent macro creates an instr_t for a 4-byte - * or 8-byte memory store instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param m The destination memory opnd. - * \param r The source register opnd. - */ -#define XINST_CREATE_store(dc, m, r) instr_create_1dst_1src(dc, OP_hint, m, r) - -/** - * This platform-independent macro creates an instr_t for a 1-byte - * memory store instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param m The destination memory opnd. - * \param r The source register opnd. - */ -#define XINST_CREATE_store_1byte(dc, m, r) instr_create_1dst_1src(dc, OP_hint, m, r) - -/** - * This platform-independent macro creates an instr_t for a 2-byte - * memory store instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param m The destination memory opnd. - * \param r The source register opnd. - */ -#define XINST_CREATE_store_2bytes(dc, m, r) instr_create_1dst_1src(dc, OP_hint, m, r) - -/** - * This platform-independent macro creates an instr_t for a register - * to register move instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The destination register opnd. - * \param s The source register opnd. - */ -#define XINST_CREATE_move(dc, d, s) instr_create_1dst_1src(dc, OP_hint, d, s) - -/** - * This platform-independent macro creates an instr_t for a multimedia - * register load instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param r The destination register opnd. - * \param m The source memory opnd. - */ -#define XINST_CREATE_load_simd(dc, r, m) instr_create_1dst_1src(dc, OP_hint, r, m) - -/** - * This platform-independent macro creates an instr_t for a multimedia - * register store instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param m The destination memory opnd. - * \param r The source register opnd. - */ -#define XINST_CREATE_store_simd(dc, m, r) instr_create_1dst_1src(dc, OP_hint, m, r) - -/** - * This platform-independent macro creates an instr_t for an indirect - * jump through memory instruction. For AArch32, the memory address - * must be aligned to 4. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param m The memory opnd holding the target. - */ -#define XINST_CREATE_jump_mem(dc, m) \ - instr_create_0dst_2src(dc, OP_hint, opnd_create_reg(DR_REG_PC), (m)) - -/** - * This platform-independent macro creates an instr_t for an indirect - * jump instruction through a register. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param r The register opnd holding the target. - */ -#define XINST_CREATE_jump_reg(dc, r) instr_create_0dst_1src(dc, OP_hint, r) - -/** - * This platform-independent macro creates an instr_t for an immediate - * integer load instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param r The destination register opnd. - * \param i The source immediate integer opnd. - */ -#define XINST_CREATE_load_int(dc, r, i) instr_create_1dst_1src(dc, OP_hint, r, i) - -/** - * This platform-independent macro creates an instr_t for a return instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - */ -#define XINST_CREATE_return(dc) instr_create_0dst_0src(dc, OP_hint) - -/** - * This platform-independent macro creates an instr_t for an unconditional - * branch instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param t The opnd_t target operand for the instruction, which can be - * either a pc (opnd_create_pc)()) or an instr_t (opnd_create_instr()). - * Be sure to ensure that the limited reach of this short branch will reach - * the target (a pc operand is not suitable for most uses unless you know - * precisely where this instruction will be encoded). - */ -#define XINST_CREATE_jump(dc, t) instr_create_0dst_1src(dc, OP_hint, t) - -/** - * This platform-independent macro creates an instr_t for an unconditional - * branch instruction with the smallest available reach. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param t The opnd_t target operand for the instruction, which can be - * either a pc (opnd_create_pc)()) or an instr_t (opnd_create_instr()). - * Be sure to ensure that the limited reach of this short branch will reach - * the target (a pc operand is not suitable for most uses unless you know - * precisely where this instruction will be encoded). - */ -#define XINST_CREATE_jump_short(dc, t) instr_create_0dst_1src(dc, OP_hint, t) - -/** - * This platform-independent macro creates an instr_t for an unconditional - * branch instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param t The opnd_t target operand for the instruction, which can be - * either a pc (opnd_create_pc)()) or an instr_t (opnd_create_instr()). - * Be sure to ensure that the limited reach of this short branch will reach - * the target (a pc operand is not suitable for most uses unless you know - * precisely where this instruction will be encoded). - */ -#define XINST_CREATE_call(dc, t) instr_create_0dst_1src(dc, OP_hint, t) - -/** - * This platform-independent macro creates an instr_t for a conditional - * branch instruction that branches if the previously-set condition codes - * indicate the condition indicated by \p pred. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param pred The #dr_pred_type_t condition to match. - * \param t The opnd_t target operand for the instruction, which can be - * either a pc (opnd_create_pc)()) or an instr_t (opnd_create_instr()). - * Be sure to ensure that the limited reach of this short branch will reach - * the target (a pc operand is not suitable for most uses unless you know - * precisely where this instruction will be encoded). - */ -#define XINST_CREATE_jump_cond(dc, pred, t) \ - (INSTR_PRED(instr_create_0dst_1src(dc, OP_hint, t), (pred))) - -/** - * This platform-independent macro creates an instr_t for an addition - * instruction that does not affect the status flags. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The opnd_t explicit destination operand for the instruction. - * \param s The opnd_t explicit source operand for the instruction. - */ -#define XINST_CREATE_add(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) - -/** - * This platform-independent macro creates an instr_t for an addition - * instruction that does not affect the status flags and takes two sources - * plus a destination. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The opnd_t explicit destination operand for the instruction. - * \param s1 The opnd_t explicit first source operand for the instruction. This - * must be a register. - * \param s2 The opnd_t explicit source operand for the instruction. This - * can be either a register or an immediate integer. - */ -#define XINST_CREATE_add_2src(dc, d, s1, s2) \ - instr_create_1dst_2src(dc, OP_hint, d, s1, s2) - -/** - * This platform-independent macro creates an instr_t for an addition - * instruction that does not affect the status flags and takes two register sources - * plus a destination, with one source being shifted logically left by - * an immediate amount that is limited to either 0, 1, 2, or 3. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The opnd_t explicit destination operand for the instruction. - * \param s1 The opnd_t explicit first source operand for the instruction. This - * must be a register. - * \param s2_toshift The opnd_t explicit source operand for the instruction. This - * must be a register. - * \param shift_amount An integer value that must be either 0, 1, 2, or 3. - */ -#define XINST_CREATE_add_sll(dc, d, s1, s2_toshift, shift_amount) \ - instr_create_1dst_3src(dc, OP_hint, d, s1, s2_toshift, \ - opnd_create_immed_int(shift_amount, OPSZ_1)) - -/** - * This platform-independent macro creates an instr_t for an addition - * instruction that does affect the status flags. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The opnd_t explicit destination operand for the instruction. - * \param s The opnd_t explicit source operand for the instruction. - */ -#define XINST_CREATE_add_s(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) - -/** - * This platform-independent macro creates an instr_t for a subtraction - * instruction that does not affect the status flags. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The opnd_t explicit destination operand for the instruction. - * \param s The opnd_t explicit source operand for the instruction. - */ -/* FIXME: How to handle a lack of flags register? */ -#define XINST_CREATE_sub(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) - -/** - * This platform-independent macro creates an instr_t for a subtraction - * instruction that does affect the status flags. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The opnd_t explicit destination operand for the instruction. - * \param s The opnd_t explicit source operand for the instruction. - */ -/* FIXME: How to handle a lack of flags register? */ -#define XINST_CREATE_sub_s(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) - -/** - * This platform-independent macro creates an instr_t for a bitwise and - * instruction that does affect the status flags. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The opnd_t explicit destination operand for the instruction. - * \param s The opnd_t explicit source operand for the instruction. - */ -/* FIXME: How to handle a lack of flags register? */ -#define XINST_CREATE_and_s(dc, d, s) instr_create_1dst_2src(dc, OP_hint, d, d, s) - -/** - * This platform-independent macro creates an instr_t for a comparison - * instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param s1 The opnd_t explicit source operand for the instruction. - * \param s2 The opnd_t explicit source operand for the instruction. - */ -/* FIXME: How to handle a lack of flags register? */ -#define XINST_CREATE_cmp(dc, s1, s2) instr_create_0dst_2src(dc, OP_hint, s1, s2) - -/** - * This platform-independent macro creates an instr_t for a software - * interrupt instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param i The source integer constant opnd_t operand. - */ -#define XINST_CREATE_interrupt(dc, i) instr_create_0dst_1src(dc, OP_hint, i) - -/** - * This platform-independent macro creates an instr_t for a logical right shift - * instruction that does affect the status flags. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param d The opnd_t explicit destination operand for the instruction. - * \param rm_or_imm The opnd_t explicit source operand for the instruction. - */ -#define XINST_CREATE_slr_s(dc, d, rm_or_imm) \ - instr_create_1dst_1src(dc, OP_hint, d, rm_or_imm) - -/** - * This platform-independent macro creates an instr_t for a nop instruction. - * \param dc The void * dcontext used to allocate memory for the instr_t. - */ -#define XINST_CREATE_nop(dc) INSTR_CREATE_nop(dc) - -/** - * This platform-independent macro creates an instr_t for an indirect call instr - * through a register. - * \param dc The void * dcontext used to allocate memory for the instr_t. - * \param r The opnd_t explicit source operand for the instruction. This should - * be a reg_id_t operand with the address of the subroutine. - */ -#define XINST_CREATE_call_reg(dc, r) instr_create_0dst_1src(dc, OP_hint, r) - -/** - * Create an absolute address operand encoded as pc-relative. - * Encoding will fail if addr is out of the maximum signed displacement - * reach for the architecture. - */ -#define OPND_CREATE_ABSMEM(addr, size) opnd_create_rel_addr(addr, size) - -/** @} */ /* end doxygen group */ - -/**************************************************************************** - * RISC-V-specific INSTR_CREATE_* macros - */ - -/** @name No-operand instructions */ -/** @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */ -/** - * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically - * supplying any implicit operands. - * \param dc The void * dcontext used to allocate memory for the instr_t. - */ -#define INSTR_CREATE_ecall(dc) instr_create_0dst_0src(dc, OP_ecall) -#define INSTR_CREATE_nop(dc) instr_create_0dst_0src(dc, OP_hint) -/** @} */ /* end doxygen group */ - -#endif /* _DR_IR_MACROS_RISCV64_H_ */ diff --git a/core/ir/riscv64/opcode_api.h b/core/ir/riscv64/opcode_api.h deleted file mode 100644 index dbc7e05f212..00000000000 --- a/core/ir/riscv64/opcode_api.h +++ /dev/null @@ -1,92 +0,0 @@ -/* ********************************************************** - * Copyright (c) 2022 Rivos, Inc. All rights reserved. - * **********************************************************/ - -/* - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * * Neither the name of Rivos, Inc. nor the names of its contributors may be - * used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL RIVOS, INC. OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH - * DAMAGE. - */ - -#ifndef _DR_IR_OPCODES_RISCV64_H_ -#define _DR_IR_OPCODES_RISCV64_H_ 1 - -/**************************************************************************** - * OPCODES - */ -/** - * @file dr_ir_opcodes_arm.h - * @brief Instruction opcode constants for RISC-V. - */ -/** Opcode constants for use in the instr_t data structure. */ -enum { - /* 0 */ OP_INVALID, - /* NULL, */ /**< INVALID opcode */ - /* 1 */ OP_UNDECODED, - /* NULL, */ /**< UNDECODED opcode */ - /* 2 */ OP_CONTD, - /* NULL, */ /**< CONTD opcode */ - /* 3 */ OP_LABEL, - /* NULL, */ /**< LABEL opcode */ - - /* FIXME i#3544: Put real opcodes here and make sure ordering matches HW. */ - OP_ecall, /**< RISC-V ecall opcode. */ - OP_l, /**< RISC-V integer load opcode. Width is encoded in operand sizes. */ - OP_s, /**< RISC-V integer store opcode. Width is encoded in operand sizes. */ - OP_add, /**< RISC-V add opcode. */ - OP_addi, /**< RISC-V addi opcode. */ - OP_sub, /**< RISC-V sub opcode. */ - OP_mul, /**< RISC-V mul opcode. */ - OP_mulu, /**< RISC-V mulu opcode. */ - OP_div, /**< RISC-V div opcode. */ - OP_divu, /**< RISC-V divu opcode. */ - OP_jal, /**< RISC-V jal opcode. */ - OP_jalr, /**< RISC-V jalr opcode. */ - OP_beq, /**< RISC-V beq opcode. */ - OP_bne, /**< RISC-V bne opcode. */ - OP_blt, /**< RISC-V blt opcode. */ - OP_bltu, /**< RISC-V bltu opcode. */ - OP_bge, /**< RISC-V bge opcode. */ - OP_bgeu, /**< RISC-V bgeu opcode. */ - OP_hint, /**< RISC-V hint opcode. */ - -#if defined(RISCV_ISA_F) || defined(RISCV_ISA_D) - OP_fl, /**< RISC-V FP load opcode. Width is encoded in operand sizes. */ - OP_fs, /**< RISC-V FP store opcode. Width is encoded in operand sizes. */ -#endif - - OP_AFTER_LAST, - /* FIXME i#3544: Replace OP_hint with a real opcode. */ - OP_FIRST = OP_hint, /**< First real opcode. */ - OP_LAST = OP_AFTER_LAST - 1, /**< Last real opcode. */ -}; - -/* alternative names */ -#define OP_jmp OP_hint /**< Platform-independent opcode name for jump. */ -#define OP_jmp_short OP_hint /**< Platform-independent opcode name for short jump. */ -#define OP_load OP_hint /**< Platform-independent opcode name for load. */ -#define OP_store OP_hint /**< Platform-independent opcode name for store. */ - -#endif /* _DR_IR_OPCODES_RISCV64_H_ */ From 25bb369df38bfd443a2c0b21189882d333a0d93d Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 00:22:12 -0400 Subject: [PATCH 15/36] Remove hook in interp and adjust logic to locate rseq_cs entries. --- core/arch/interp.c | 14 -------------- core/unix/os.c | 4 ++-- core/unix/rseq_linux.c | 30 ++++++++++++++++++++---------- core/unix/rseq_linux.h | 2 +- 4 files changed, 23 insertions(+), 27 deletions(-) diff --git a/core/arch/interp.c b/core/arch/interp.c index efecdebf9ef..fa51eef80c5 100644 --- a/core/arch/interp.c +++ b/core/arch/interp.c @@ -66,9 +66,6 @@ #include "../native_exec.h" #include "../jit_opt.h" -#if defined(LINUX) -# include "../unix/rseq_linux.h" -#endif #ifdef CHECK_RETURNS_SSE2 # include /* for warning when see libc setjmp */ #endif @@ -376,17 +373,6 @@ set_reached_image_entry() SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); reached_image_entry = true; SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); -#if defined(LINUX) - if (!dynamo_control_via_attach) { - /* For enabling rseq in the non-attach case, we need to wait till libc is - * initialized. This is a good control point. - * Note that d_r_rseq_init would already have completed by now. - */ - rseq_check_glibc_enabled(); - if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(false); - } -#endif } bool diff --git a/core/unix/os.c b/core/unix/os.c index a73d0979b5e..538396f8841 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -10729,7 +10729,7 @@ os_take_over_all_unknown_threads(dcontext_t *dcontext) #ifdef LINUX /* Check this thread for rseq in between setup and start. */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(false); + rseq_locate_rseq_regions(); #endif /* Find tids for which we have no thread record, meaning they are not under @@ -10957,7 +10957,7 @@ os_thread_take_over(priv_mcontext_t *mc, kernel_sigset_t *sigset) * regions as rseq when the rseq syscall is never set up. */ if (rseq_is_registered_for_current_thread()) { - rseq_locate_rseq_regions(false); + rseq_locate_rseq_regions(); rseq_thread_attach(dcontext); } #endif diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 52c9b49c3a9..c60a7cbcc5c 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -177,18 +177,21 @@ rseq_check_glibc_enabled() if (rseq_offset_addr == NULL || *rseq_offset_addr == 0) continue; /* Verify that the offset has not changed. */ - ASSERT(glibc_rseq_offset == 0 || - (glibc_rseq_offset == *rseq_offset_addr && - rseq_tls_offset == *rseq_offset_addr)); + ASSERT(glibc_rseq_offset == 0 || glibc_rseq_offset == *rseq_offset_addr); + SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, *rseq_offset_addr, false); - ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); + SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); uint *rseq_size_addr = (uint *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_size"); ASSERT(rseq_size_addr != NULL); /* Glibc's rseq support is disabled. */ if (*rseq_size_addr == 0) continue; + ASSERT(rseq_tls_offset == 0 || rseq_tls_offset == *rseq_offset_addr); + SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); + ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); ATOMIC_4BYTE_WRITE(&glibc_rseq_size, *rseq_size_addr, false); + SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); DODEBUG({ byte *rseq_addr = get_app_segment_base(LIB_SEG_TLS) + glibc_rseq_offset; int res = @@ -227,11 +230,13 @@ d_r_rseq_exit(void) DELETE_LOCK(rseq_trigger_lock); /* Better to reset and detect again for reattaches. */ uint new_value = 0; + SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); ATOMIC_4BYTE_WRITE(&rseq_tls_offset, new_value, false); #ifdef GLIBC_RSEQ ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, new_value, false); ATOMIC_4BYTE_WRITE(&glibc_rseq_size, new_value, false); #endif + SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); } void @@ -535,7 +540,12 @@ rseq_process_elf_sections(module_area_t *ma, bool at_map, * relocated and only need the offset if relocations have not yet been applied. */ ssize_t entry_offs = 0; - if (at_map || (DYNAMO_OPTION(early_inject) && !dr_api_entry && !dynamo_started)) + bool adjust_entry_offs = + at_map || (DYNAMO_OPTION(early_inject) && !dr_api_entry && !dynamo_started); +#ifdef GLIBC_RSEQ + adjust_entry_offs |= (glibc_rseq_offset == 0); +#endif + if (adjust_entry_offs) entry_offs = load_offs; for (i = 0; i < elf_hdr->e_shnum; i++) { #define RSEQ_PTR_ARRAY_SEC_NAME "__rseq_cs_ptr_array" @@ -697,7 +707,7 @@ rseq_locate_tls_offset(void) #ifdef GLIBC_RSEQ /* If Glibc's rseq support is enabled, we already know the offset. */ if (glibc_rseq_size > 0) { - ASSERT(rseq_tls_offset > 0); + ASSERT(rseq_tls_offset > 0 && glibc_rseq_offset == rseq_tls_offset); return rseq_tls_offset; } #endif @@ -803,11 +813,11 @@ rseq_locate_rseq_regions() return; #if defined(GLIBC_RSEQ) rseq_check_glibc_enabled(); - /* Glibc's rseq support has not been initialized yet. Return for now and - * try again later. + /* We should either have determined availability of glibc's rseq support (so that + * rseq_locate_tls_offset works as expected, or should already have seen an + * rseq syscall (which would set rseq_tls_offset). */ - if (glibc_rseq_offset == 0) - return; + ASSERT(glibc_rseq_offset > 0 || rseq_tls_offset > 0); #endif /* This is a global operation, but the trigger could be hit by two threads at once, * thus requiring synchronization. diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index 71a61f7bf99..9f4084cc373 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -62,7 +62,7 @@ bool rseq_is_registered_for_current_thread(void); void -rseq_locate_rseq_regions(); +rseq_locate_rseq_regions(void); void rseq_module_init(module_area_t *ma, bool at_map); From 71d488142b87bc57f34a393b3a958dae8d8b97aa Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 13:18:47 -0400 Subject: [PATCH 16/36] Cleanup. --- core/unix/os.c | 5 ----- core/unix/rseq_linux.c | 22 +++++++++++++--------- core/unix/rseq_linux.h | 3 --- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/core/unix/os.c b/core/unix/os.c index 538396f8841..f8d3ea94d98 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -10067,11 +10067,6 @@ find_executable_vm_areas(void) /* now that we have the modules set up, query libc */ get_libc_errno_location(true /*force init*/); -#if defined(LINUX) - /* For rseq on GLIBC >= 2.35, we need to find __rseq_size and __rseq_offset, - * but we cannot do that here as they wouldn't be initialized yet. - */ -#endif return count; } diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index c60a7cbcc5c..1a35e9cac5b 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -75,9 +75,10 @@ struct rseq { #ifdef GLIBC_RSEQ /* __tls_init_tp in glibc/sysdeps/nptl/dl-tls_init_tp.c initializes __rseq_size - * and __rseq_offset. The former is > 0 iff glibc's rseq support is not disabled - * using GLIBC_TUNABLES=glibc.pthread.rseq=0. The latter is defined regardless - * and is used by us to determine whether glibc has been initialized yet. + * and __rseq_offset. The former is > 0 iff glibc's rseq support is enabled + * (that is, not disabled by the user using GLIBC_TUNABLES=glibc.pthread.rseq=0 + * or due to some issue). The latter is defined regardless and is used by us to + * determine whether glibc has been initialized yet. */ static volatile uint glibc_rseq_size; static volatile int glibc_rseq_offset; @@ -162,7 +163,7 @@ rseq_cs_free(dcontext_t *dcontext, void *data) } void -rseq_check_glibc_enabled() +rseq_check_glibc_support() { #ifdef GLIBC_RSEQ /* We have already determined Glibc's rseq support. */ @@ -173,7 +174,7 @@ rseq_check_glibc_enabled() module_area_t *ma = module_iterator_next(iter); int *rseq_offset_addr = (int *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_offset"); - /* Glibc's not initialized yet. */ + /* Glibc is not completely initialized yet. */ if (rseq_offset_addr == NULL || *rseq_offset_addr == 0) continue; /* Verify that the offset has not changed. */ @@ -181,6 +182,7 @@ rseq_check_glibc_enabled() SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, *rseq_offset_addr, false); SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); + uint *rseq_size_addr = (uint *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_size"); ASSERT(rseq_size_addr != NULL); @@ -192,6 +194,7 @@ rseq_check_glibc_enabled() ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); ATOMIC_4BYTE_WRITE(&glibc_rseq_size, *rseq_size_addr, false); SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); + DODEBUG({ byte *rseq_addr = get_app_segment_base(LIB_SEG_TLS) + glibc_rseq_offset; int res = @@ -216,7 +219,7 @@ d_r_rseq_init(void) rseq_cs_table = generic_hash_create(GLOBAL_DCONTEXT, INIT_RSEQ_CS_TABLE_SIZE, 80, HASHTABLE_SHARED | HASHTABLE_PERSISTENT, rseq_cs_free _IF_DEBUG("rseq_cs table")); - rseq_check_glibc_enabled(); + rseq_check_glibc_support(); /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ if (rseq_is_registered_for_current_thread()) rseq_locate_rseq_regions(); @@ -376,7 +379,7 @@ bool rseq_is_registered_for_current_thread(void) { #ifdef GLIBC_RSEQ - /* If Glibc's rseq support is enabled, the struct rseq is already + /* If Glibc's rseq support is enabled, a struct rseq is already * registered for all threads. */ if (glibc_rseq_size > 0) @@ -543,6 +546,7 @@ rseq_process_elf_sections(module_area_t *ma, bool at_map, bool adjust_entry_offs = at_map || (DYNAMO_OPTION(early_inject) && !dr_api_entry && !dynamo_started); #ifdef GLIBC_RSEQ + /* We want to set entry_offs if glibc has not been initialized yet. */ adjust_entry_offs |= (glibc_rseq_offset == 0); #endif if (adjust_entry_offs) @@ -807,12 +811,12 @@ rseq_process_syscall(dcontext_t *dcontext) * the app has registered the current thread for rseq. */ void -rseq_locate_rseq_regions() +rseq_locate_rseq_regions(void) { if (rseq_enabled) return; #if defined(GLIBC_RSEQ) - rseq_check_glibc_enabled(); + rseq_check_glibc_support(); /* We should either have determined availability of glibc's rseq support (so that * rseq_locate_tls_offset works as expected, or should already have seen an * rseq syscall (which would set rseq_tls_offset). diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index 9f4084cc373..15c326b74dd 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -70,7 +70,4 @@ rseq_module_init(module_area_t *ma, bool at_map); void rseq_process_syscall(dcontext_t *dcontext); -void -rseq_check_glibc_enabled(void); - #endif /* _RSEQ_H_ */ From 782c8118591f3ea37768a4b58bdf6d6605f6033a Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 13:23:09 -0400 Subject: [PATCH 17/36] Revert "Revert "Revert "Add tmate""" This reverts commit 64cb13188abe09eec52e238b96c25a9edd923f52. --- .github/workflows/ci-x86.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci-x86.yml b/.github/workflows/ci-x86.yml index 11416c64f50..f2b4da11f8b 100644 --- a/.github/workflows/ci-x86.yml +++ b/.github/workflows/ci-x86.yml @@ -172,13 +172,6 @@ jobs: CI_TRIGGER: ${{ github.event_name }} CI_BRANCH: ${{ github.ref }} - - name: Setup tmate session on failure - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - - - name: Setup tmate session on success - uses: mxschmitt/action-tmate@v3 - - name: Send failure mail to dynamorio-devs if: failure() && github.ref == 'refs/heads/master' uses: dawidd6/action-send-mail@v2 From c418d74ec63f6b9f2e5a7ec12372137509418b0e Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 14:10:59 -0400 Subject: [PATCH 18/36] Detect app's glibc instead of relying on the macro. --- core/unix/rseq_linux.c | 54 +++++++++++++++++++++--------------------- core/unix/rseq_linux.h | 4 ---- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 1a35e9cac5b..da4800c2fe1 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -73,16 +73,16 @@ struct rseq { } __attribute__((aligned(4 * sizeof(uint64)))); #define RSEQ_FLAG_UNREGISTER 1 -#ifdef GLIBC_RSEQ +/* Could be that the app is using an older glibc, or no glibc at all. */ +static volatile bool have_glibc_rseq; /* __tls_init_tp in glibc/sysdeps/nptl/dl-tls_init_tp.c initializes __rseq_size * and __rseq_offset. The former is > 0 iff glibc's rseq support is enabled * (that is, not disabled by the user using GLIBC_TUNABLES=glibc.pthread.rseq=0 - * or due to some issue). The latter is defined regardless and is used by us to - * determine whether glibc has been initialized yet. + * or due to some issue). The latter is initialized regardless and is used by + * us to determine whether glibc has been initialized completely yet. */ static volatile uint glibc_rseq_size; static volatile int glibc_rseq_offset; -#endif vm_area_vector_t *d_r_rseq_areas; DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, @@ -165,7 +165,6 @@ rseq_cs_free(dcontext_t *dcontext, void *data) void rseq_check_glibc_support() { -#ifdef GLIBC_RSEQ /* We have already determined Glibc's rseq support. */ if (glibc_rseq_offset > 0) return; @@ -174,8 +173,14 @@ rseq_check_glibc_support() module_area_t *ma = module_iterator_next(iter); int *rseq_offset_addr = (int *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_offset"); + if (rseq_offset_addr == NULL) + continue; + SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); + ATOMIC_1BYTE_WRITE(&have_glibc_rseq, true, false); + SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); + /* Glibc is not completely initialized yet. */ - if (rseq_offset_addr == NULL || *rseq_offset_addr == 0) + if (*rseq_offset_addr == 0) continue; /* Verify that the offset has not changed. */ ASSERT(glibc_rseq_offset == 0 || glibc_rseq_offset == *rseq_offset_addr); @@ -187,8 +192,10 @@ rseq_check_glibc_support() (uint *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_size"); ASSERT(rseq_size_addr != NULL); /* Glibc's rseq support is disabled. */ - if (*rseq_size_addr == 0) + if (*rseq_size_addr == 0) { + ASSERT(glibc_rseq_size == 0); continue; + } ASSERT(rseq_tls_offset == 0 || rseq_tls_offset == *rseq_offset_addr); SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); @@ -206,7 +213,6 @@ rseq_check_glibc_support() }); } module_iterator_stop(iter); -#endif } void @@ -232,13 +238,13 @@ d_r_rseq_exit(void) vmvector_delete_vector(GLOBAL_DCONTEXT, d_r_rseq_areas); DELETE_LOCK(rseq_trigger_lock); /* Better to reset and detect again for reattaches. */ - uint new_value = 0; SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); - ATOMIC_4BYTE_WRITE(&rseq_tls_offset, new_value, false); -#ifdef GLIBC_RSEQ - ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, new_value, false); - ATOMIC_4BYTE_WRITE(&glibc_rseq_size, new_value, false); -#endif + ATOMIC_4BYTE_WRITE(&rseq_tls_offset, 0, false); + if (have_glibc_rseq) { + ATOMIC_1BYTE_WRITE(&have_glibc_rseq, false, false); + ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, 0, false); + ATOMIC_4BYTE_WRITE(&glibc_rseq_size, 0, false); + } SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); } @@ -378,13 +384,11 @@ rseq_shared_fragment_flushtime_update(dcontext_t *dcontext) bool rseq_is_registered_for_current_thread(void) { -#ifdef GLIBC_RSEQ /* If Glibc's rseq support is enabled, a struct rseq is already * registered for all threads. */ if (glibc_rseq_size > 0) return true; -#endif /* Unfortunately there's no way to query the current rseq struct. * For 64-bit we can pass a kernel address and look for EFAULT * vs EINVAL, but there is no kernel address for 32-bit. @@ -545,10 +549,10 @@ rseq_process_elf_sections(module_area_t *ma, bool at_map, ssize_t entry_offs = 0; bool adjust_entry_offs = at_map || (DYNAMO_OPTION(early_inject) && !dr_api_entry && !dynamo_started); -#ifdef GLIBC_RSEQ - /* We want to set entry_offs if glibc has not been initialized yet. */ - adjust_entry_offs |= (glibc_rseq_offset == 0); -#endif + if (have_glibc_rseq) { + /* We want to set entry_offs if glibc has not been initialized yet. */ + adjust_entry_offs |= (glibc_rseq_offset == 0); + } if (adjust_entry_offs) entry_offs = load_offs; for (i = 0; i < elf_hdr->e_shnum; i++) { @@ -708,13 +712,11 @@ rseq_process_module(module_area_t *ma, bool at_map) static int rseq_locate_tls_offset(void) { -#ifdef GLIBC_RSEQ /* If Glibc's rseq support is enabled, we already know the offset. */ if (glibc_rseq_size > 0) { ASSERT(rseq_tls_offset > 0 && glibc_rseq_offset == rseq_tls_offset); return rseq_tls_offset; } -#endif /* We assume (and document) that the loader's static TLS is used, so every thread * has a consistent %fs:-offs address. Unfortunately, using a local copy of the * rseq code for our non-instrumented execution requires us to locate the app's @@ -815,14 +817,12 @@ rseq_locate_rseq_regions(void) { if (rseq_enabled) return; -#if defined(GLIBC_RSEQ) rseq_check_glibc_support(); - /* We should either have determined availability of glibc's rseq support (so that - * rseq_locate_tls_offset works as expected, or should already have seen an + /* We should either have already determined availability of glibc's rseq support + * (so that rseq_locate_tls_offset properly), or should already have seen an * rseq syscall (which would set rseq_tls_offset). */ - ASSERT(glibc_rseq_offset > 0 || rseq_tls_offset > 0); -#endif + ASSERT(!have_glibc_rseq || glibc_rseq_offset > 0 || rseq_tls_offset > 0); /* This is a global operation, but the trigger could be hit by two threads at once, * thus requiring synchronization. */ diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index 15c326b74dd..56a25aa4dd9 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -45,10 +45,6 @@ # error Rseq is a Linux-only feature #endif -#if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 35) -# define GLIBC_RSEQ 1 -#endif - void d_r_rseq_init(void); From 8c2744aed6a41cd8b377d0a99c49ee05ebd815e4 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 14:43:01 -0400 Subject: [PATCH 19/36] Move check_glibc_rseq_support after lock --- core/unix/rseq_linux.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index da4800c2fe1..4e2d672a716 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -817,6 +817,11 @@ rseq_locate_rseq_regions(void) { if (rseq_enabled) return; + d_r_mutex_lock(&rseq_trigger_lock); + if (rseq_enabled) { + d_r_mutex_unlock(&rseq_trigger_lock); + return; + } rseq_check_glibc_support(); /* We should either have already determined availability of glibc's rseq support * (so that rseq_locate_tls_offset properly), or should already have seen an @@ -826,12 +831,6 @@ rseq_locate_rseq_regions(void) /* This is a global operation, but the trigger could be hit by two threads at once, * thus requiring synchronization. */ - d_r_mutex_lock(&rseq_trigger_lock); - if (rseq_enabled) { - d_r_mutex_unlock(&rseq_trigger_lock); - return; - } - int offset = 0; if (rseq_tls_offset == 0) { /* Identify the TLS offset of this thread's struct rseq. */ From 1789ef469f84f447600ede99d694474e6cd6db96 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 15:15:42 -0400 Subject: [PATCH 20/36] Add ubuntu-22 test filters back. --- suite/runsuite.cmake | 13 +++++++++++++ suite/tests/CMakeLists.txt | 28 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/suite/runsuite.cmake b/suite/runsuite.cmake index 2d02c2a945e..1e6040628bf 100644 --- a/suite/runsuite.cmake +++ b/suite/runsuite.cmake @@ -77,6 +77,19 @@ foreach (arg ${CTEST_SCRIPT_ARG}) endif () endforeach (arg) +if (UNIX AND NOT APPLE AND NOT ANDROID) + execute_process(COMMAND ldd --version + RESULT_VARIABLE ldd_result ERROR_VARIABLE ldd_err OUTPUT_VARIABLE ldd_out) + if (ldd_result OR ldd_err) + # Failed; just move on. + elseif (ldd_out MATCHES "GLIBC 2.3[5-9]") + # XXX i#5437, i#5431: While we work through Ubuntu22 issues we run + # just a few tests. + set(extra_ctest_args INCLUDE_LABEL UBUNTU_22) + set(arg_debug_only ON) + endif () +endif () + set(build_tests "BUILD_TESTS:BOOL=ON") if (arg_automated_ci) diff --git a/suite/tests/CMakeLists.txt b/suite/tests/CMakeLists.txt index a7309c33d77..12558f99638 100644 --- a/suite/tests/CMakeLists.txt +++ b/suite/tests/CMakeLists.txt @@ -5355,3 +5355,31 @@ if (NOT ANDROID AND AARCHXX) PROPERTIES LABELS RUNS_ON_QEMU) endif () endif (NOT ANDROID AND AARCHXX) + +# XXX i#5437, i#5431: While we work through Ubuntu22 issues we want our new GA CI job to +# just run sanity tests that do pass now as a glibc change detector to see if our +# workarounds so far break over time. +set_tests_properties( + code_api|common.broadfun + PROPERTIES LABELS UBUNTU_22) +if (BUILD_SAMPLES AND NOT ANDROID) + set_tests_properties( + code_api|sample.bbsize + PROPERTIES LABELS UBUNTU_22) +endif () +if (BUILD_CLIENTS AND NOT ANDROID) + set_tests_properties( + code_api|tool.drcachesim.simple + code_api|tool.drcacheoff.simple + PROPERTIES LABELS UBUNTU_22) +endif () +if (LINUX AND X64 AND HAVE_RSEQ) + set_tests_properties( + code_api|api.rseq + code_api|linux.rseq_disable + code_api|linux.rseq_noarray + code_api|linux.rseq + code_api|linux.rseq_table + code_api|tool.drcacheoff.rseq + PROPERTIES LABELS UBUNTU_22) +endif () From b57460bfedb6e817f8d692a4029b4d0725515da5 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 18:58:27 -0400 Subject: [PATCH 21/36] Cleanup. --- core/dynamo.c | 2 ++ core/unix/rseq_linux.c | 13 +++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/dynamo.c b/core/dynamo.c index d54552b6ce6..815741ea7d3 100644 --- a/core/dynamo.c +++ b/core/dynamo.c @@ -695,6 +695,7 @@ dynamorio_app_init_part_two_finalize(void) find_dynamo_library_vm_areas(); dynamo_vm_areas_unlock(); } + #ifdef LINUX /* We need to delay initializing rseq support to after we have * initialized modules. @@ -702,6 +703,7 @@ dynamorio_app_init_part_two_finalize(void) if (!standalone_library) d_r_rseq_init(); #endif + #ifdef ANNOTATIONS annotation_init(); #endif diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 4e2d672a716..2576784724b 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -817,20 +817,21 @@ rseq_locate_rseq_regions(void) { if (rseq_enabled) return; + /* This is a global operation, but the trigger could be hit by two threads at once, + * thus requiring synchronization. + */ d_r_mutex_lock(&rseq_trigger_lock); if (rseq_enabled) { d_r_mutex_unlock(&rseq_trigger_lock); return; } rseq_check_glibc_support(); - /* We should either have already determined availability of glibc's rseq support - * (so that rseq_locate_tls_offset properly), or should already have seen an - * rseq syscall (which would set rseq_tls_offset). + /* If the app's Glibc version supports rseq, then we should either have already + * determined availability of Glibc's rseq support (for rseq_locate_tls_offset + * to work as intended), or should already have seen an rseq syscall (which + * would set rseq_tls_offset). */ ASSERT(!have_glibc_rseq || glibc_rseq_offset > 0 || rseq_tls_offset > 0); - /* This is a global operation, but the trigger could be hit by two threads at once, - * thus requiring synchronization. - */ int offset = 0; if (rseq_tls_offset == 0) { /* Identify the TLS offset of this thread's struct rseq. */ From 599c8cc892e43d8b866e9368ed9a4be7cf14be32 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 19:43:03 -0400 Subject: [PATCH 22/36] Fix rseq registration in rseq_thread_loop --- suite/tests/linux/rseq.c | 60 +++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index fd8ce3be2ab..1b2984552c8 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -95,11 +95,6 @@ # define RSEQ_ADD_ARRAY_ENTRY(label) /* Nothing. */ #endif -static int rseq_tls_offset; - -#ifdef GLIBC_RSEQ -static bool glibc_rseq_enabled; -#endif /* This cannot be a stack-local variable, as the kernel will force SIGSEGV * if it can't read this struct. And for multiple threads it should be in TLS. */ @@ -117,12 +112,34 @@ static void *thread_ready; static volatile int sigill_count; +static bool +register_rseq() +{ +#ifdef GLIBC_RSEQ + if (__rseq_size > 0) { + /* Already registered by Glibc. Assert that it's there. */ + assert(__rseq_offset > 0); + struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; + int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, 0); + assert(res == -1 && errno == EPERM); + } else { +#endif + rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; + int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); + assert(res == 0); +#ifdef GLIBC_RSEQ + } +#endif + /* Linux kernel 4.18+ is required. */ + return errno != ENOSYS; +} + static volatile struct rseq * get_my_rseq() { #ifdef GLIBC_RSEQ - if (glibc_rseq_enabled) - return (struct rseq *)(__builtin_thread_pointer() + rseq_tls_offset); + if (__rseq_size > 0) + return (struct rseq *)(__builtin_thread_pointer() + __rseq_offset); #endif return &rseq_tls; } @@ -829,11 +846,9 @@ rseq_thread_loop(void *arg) * in this function is close enough: the test already has non-determinism. */ signal_cond_var(thread_ready); + bool res = register_rseq(); + assert(res); volatile struct rseq *reg_rseq = get_my_rseq(); - reg_rseq->cpu_id = RSEQ_CPU_ID_UNINITIALIZED; - int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, RSEQ_SIG); - if (res != 0) - return NULL; static int zero; # ifdef X86 __asm__ __volatile__( @@ -1001,24 +1016,8 @@ int main() { intercept_signal(SIGILL, signal_handler, false); - int res; - int expected_errno; -#ifdef GLIBC_RSEQ - glibc_rseq_enabled = __rseq_size > 0; - if (glibc_rseq_enabled) { - assert(__rseq_offset > 0); - struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; - res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, 0); - expected_errno = EPERM; - } else { -#endif - rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; - res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); - expected_errno = 0; -#ifdef GLIBC_RSEQ - } -#endif - if (errno == expected_errno) { + bool res = register_rseq(); + if (res) { #ifdef RSEQ_TEST_ATTACH /* Set -offline to avoid trying to open a pipe to a missing reader. */ if (setenv("DYNAMORIO_OPTIONS", "-stderr_mask 0xc -client_lib ';;-offline'", @@ -1054,9 +1053,6 @@ main() join_thread(mythread); destroy_cond_var(thread_ready); #endif - } else { - /* Linux kernel 4.18+ is required. */ - assert(errno == ENOSYS); } print("All done\n"); return 0; From 9e059ab9bcc242f43a4f79759f57acd53da8fca4 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Tue, 1 Nov 2022 22:15:25 -0400 Subject: [PATCH 23/36] Cleanup. --- core/dynamo.c | 4 ++-- core/unix/rseq_linux.c | 19 ++++++++++--------- suite/tests/linux/rseq.c | 4 ++-- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/core/dynamo.c b/core/dynamo.c index 815741ea7d3..6cc183c1312 100644 --- a/core/dynamo.c +++ b/core/dynamo.c @@ -697,8 +697,8 @@ dynamorio_app_init_part_two_finalize(void) } #ifdef LINUX - /* We need to delay initializing rseq support to after we have - * initialized modules. + /* Rseq support should be initialized only after modules have been + * initialized too. */ if (!standalone_library) d_r_rseq_init(); diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 2576784724b..e9af86f358d 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -73,7 +73,9 @@ struct rseq { } __attribute__((aligned(4 * sizeof(uint64)))); #define RSEQ_FLAG_UNREGISTER 1 -/* Could be that the app is using an older glibc, or no glibc at all. */ +/* Could be that the app is using glibc older than 2.35 (which did not provide + * rseq support), or no glibc at all. + */ static volatile bool have_glibc_rseq; /* __tls_init_tp in glibc/sysdeps/nptl/dl-tls_init_tp.c initializes __rseq_size * and __rseq_offset. The former is > 0 iff glibc's rseq support is enabled @@ -165,7 +167,7 @@ rseq_cs_free(dcontext_t *dcontext, void *data) void rseq_check_glibc_support() { - /* We have already determined Glibc's rseq support. */ + /* We have already determined glibc's rseq support. */ if (glibc_rseq_offset > 0) return; module_iterator_t *iter = module_iterator_start(); @@ -384,7 +386,7 @@ rseq_shared_fragment_flushtime_update(dcontext_t *dcontext) bool rseq_is_registered_for_current_thread(void) { - /* If Glibc's rseq support is enabled, a struct rseq is already + /* If glibc's rseq support is enabled, a struct rseq is already * registered for all threads. */ if (glibc_rseq_size > 0) @@ -712,7 +714,7 @@ rseq_process_module(module_area_t *ma, bool at_map) static int rseq_locate_tls_offset(void) { - /* If Glibc's rseq support is enabled, we already know the offset. */ + /* If glibc's rseq support is enabled, we already know the offset. */ if (glibc_rseq_size > 0) { ASSERT(rseq_tls_offset > 0 && glibc_rseq_offset == rseq_tls_offset); return rseq_tls_offset; @@ -826,12 +828,11 @@ rseq_locate_rseq_regions(void) return; } rseq_check_glibc_support(); - /* If the app's Glibc version supports rseq, then we should either have already - * determined availability of Glibc's rseq support (for rseq_locate_tls_offset - * to work as intended), or should already have seen an rseq syscall (which - * would set rseq_tls_offset). + /* If the app's glibc version supports rseq, then we should have already + * determined rseq_tls_offset, either by rseq_check_glibc_support or + * rseq_process_syscall (for non-attach cases). */ - ASSERT(!have_glibc_rseq || glibc_rseq_offset > 0 || rseq_tls_offset > 0); + ASSERT(glibc_rseq_size == 0 || rseq_tls_offset > 0); int offset = 0; if (rseq_tls_offset == 0) { /* Identify the TLS offset of this thread's struct rseq. */ diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index 1b2984552c8..3613d187bcf 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -58,7 +58,7 @@ # define _GNU_SOURCE #endif #include -#if defined(__GLIBC__) && ((__GLIBC__ > 2) || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 35)) +#if defined(__GLIBC__) && (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 35)) # include # define GLIBC_RSEQ 1 #else @@ -117,7 +117,7 @@ register_rseq() { #ifdef GLIBC_RSEQ if (__rseq_size > 0) { - /* Already registered by Glibc. Assert that it's there. */ + /* Already registered by glibc. Verify that it's there. */ assert(__rseq_offset > 0); struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, 0); From 1b69dad8bae9244fefe79467d6ffdfbe4a6bf743 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Wed, 2 Nov 2022 09:25:15 -0400 Subject: [PATCH 24/36] Fix ENOSYS assertion in tests. --- suite/tests/linux/rseq.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index 3613d187bcf..16fd1e2321d 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -121,12 +121,12 @@ register_rseq() assert(__rseq_offset > 0); struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, 0); - assert(res == -1 && errno == EPERM); + assert(res == -1 && (errno == EPERM || errno == ENOSYS)); } else { #endif rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; int res = syscall(SYS_rseq, &rseq_tls, sizeof(rseq_tls), 0, RSEQ_SIG); - assert(res == 0); + assert(res == 0 || (res == -1 && errno == ENOSYS)); #ifdef GLIBC_RSEQ } #endif From 62e5908746cf1b45c9c0738ef6e7e7e5f2bd7cb5 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Thu, 10 Nov 2022 10:43:11 -0500 Subject: [PATCH 25/36] Revert previous design since it doesn't work on fully static binaries. --- core/dynamo.c | 15 ------ core/unix/os.c | 8 +++ core/unix/rseq_linux.c | 108 ++--------------------------------------- 3 files changed, 12 insertions(+), 119 deletions(-) diff --git a/core/dynamo.c b/core/dynamo.c index 6cc183c1312..d534321d32b 100644 --- a/core/dynamo.c +++ b/core/dynamo.c @@ -92,9 +92,6 @@ portable and to avoid frequency scaling." # endif #endif -#ifdef LINUX -# include "rseq_linux.h" -#endif /* global thread-shared variables */ bool dynamo_initialized = false; static bool dynamo_options_initialized = false; @@ -696,14 +693,6 @@ dynamorio_app_init_part_two_finalize(void) dynamo_vm_areas_unlock(); } -#ifdef LINUX - /* Rseq support should be initialized only after modules have been - * initialized too. - */ - if (!standalone_library) - d_r_rseq_init(); -#endif - #ifdef ANNOTATIONS annotation_init(); #endif @@ -1179,10 +1168,6 @@ dynamo_shared_exit(thread_record_t *toexit /* must ==cur thread for Linux */ os_fast_exit(); os_slow_exit(); native_exec_exit(); /* before vm_areas_exit for using dynamo_areas */ -#ifdef LINUX - if (!standalone_library) - d_r_rseq_exit(); -#endif vm_areas_exit(); perscache_slow_exit(); /* fast called in dynamo_process_exit_with_thread_info() */ modules_exit(); /* after aslr_exit() from os_slow_exit(), diff --git a/core/unix/os.c b/core/unix/os.c index f8d3ea94d98..ffb1f5649d6 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -1031,6 +1031,10 @@ d_r_os_init(void) */ init_android_version(); #endif +#ifdef LINUX + if (!standalone_library) + d_r_rseq_init(); +#endif #ifdef MACOS64 tls_process_init(); #endif @@ -1437,6 +1441,10 @@ os_slow_exit(void) { #ifdef MACOS64 tls_process_exit(); +#endif +#ifdef LINUX + if (!standalone_library) + d_r_rseq_exit(); #endif d_r_signal_exit(); memquery_exit(); diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index e9af86f358d..c0c691644e9 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -73,26 +73,13 @@ struct rseq { } __attribute__((aligned(4 * sizeof(uint64)))); #define RSEQ_FLAG_UNREGISTER 1 -/* Could be that the app is using glibc older than 2.35 (which did not provide - * rseq support), or no glibc at all. - */ -static volatile bool have_glibc_rseq; -/* __tls_init_tp in glibc/sysdeps/nptl/dl-tls_init_tp.c initializes __rseq_size - * and __rseq_offset. The former is > 0 iff glibc's rseq support is enabled - * (that is, not disabled by the user using GLIBC_TUNABLES=glibc.pthread.rseq=0 - * or due to some issue). The latter is initialized regardless and is used by - * us to determine whether glibc has been initialized completely yet. - */ -static volatile uint glibc_rseq_size; -static volatile int glibc_rseq_offset; - vm_area_vector_t *d_r_rseq_areas; DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, INIT_LOCK_FREE(rseq_trigger_lock)); static volatile bool rseq_enabled; /* We require all threads to use the same TLS offset to point at struct rseq. */ -static volatile int rseq_tls_offset; +static int rseq_tls_offset; /* The signature is registered per thread, but we require all registrations * to be the same. @@ -164,59 +151,6 @@ rseq_cs_free(dcontext_t *dcontext, void *data) } while (record != NULL); } -void -rseq_check_glibc_support() -{ - /* We have already determined glibc's rseq support. */ - if (glibc_rseq_offset > 0) - return; - module_iterator_t *iter = module_iterator_start(); - while (module_iterator_hasnext(iter)) { - module_area_t *ma = module_iterator_next(iter); - int *rseq_offset_addr = - (int *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_offset"); - if (rseq_offset_addr == NULL) - continue; - SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); - ATOMIC_1BYTE_WRITE(&have_glibc_rseq, true, false); - SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); - - /* Glibc is not completely initialized yet. */ - if (*rseq_offset_addr == 0) - continue; - /* Verify that the offset has not changed. */ - ASSERT(glibc_rseq_offset == 0 || glibc_rseq_offset == *rseq_offset_addr); - SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); - ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, *rseq_offset_addr, false); - SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); - - uint *rseq_size_addr = - (uint *)dr_get_proc_address((module_handle_t)ma->start, "__rseq_size"); - ASSERT(rseq_size_addr != NULL); - /* Glibc's rseq support is disabled. */ - if (*rseq_size_addr == 0) { - ASSERT(glibc_rseq_size == 0); - continue; - } - ASSERT(rseq_tls_offset == 0 || rseq_tls_offset == *rseq_offset_addr); - SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); - ATOMIC_4BYTE_WRITE(&rseq_tls_offset, *rseq_offset_addr, false); - ATOMIC_4BYTE_WRITE(&glibc_rseq_size, *rseq_size_addr, false); - SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); - - DODEBUG({ - byte *rseq_addr = get_app_segment_base(LIB_SEG_TLS) + glibc_rseq_offset; - int res = - dynamorio_syscall(SYS_rseq, 4, rseq_addr, sizeof(struct rseq), 0, 0); - ASSERT(res == -EPERM); - LOG(GLOBAL, LOG_LOADER, 2, - "Found struct rseq registered by glibc @ " PFX " at TLS offset %d\n", - rseq_addr, rseq_tls_offset); - }); - } - module_iterator_stop(iter); -} - void d_r_rseq_init(void) { @@ -227,7 +161,6 @@ d_r_rseq_init(void) rseq_cs_table = generic_hash_create(GLOBAL_DCONTEXT, INIT_RSEQ_CS_TABLE_SIZE, 80, HASHTABLE_SHARED | HASHTABLE_PERSISTENT, rseq_cs_free _IF_DEBUG("rseq_cs table")); - rseq_check_glibc_support(); /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ if (rseq_is_registered_for_current_thread()) rseq_locate_rseq_regions(); @@ -239,15 +172,6 @@ d_r_rseq_exit(void) generic_hash_destroy(GLOBAL_DCONTEXT, rseq_cs_table); vmvector_delete_vector(GLOBAL_DCONTEXT, d_r_rseq_areas); DELETE_LOCK(rseq_trigger_lock); - /* Better to reset and detect again for reattaches. */ - SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); - ATOMIC_4BYTE_WRITE(&rseq_tls_offset, 0, false); - if (have_glibc_rseq) { - ATOMIC_1BYTE_WRITE(&have_glibc_rseq, false, false); - ATOMIC_4BYTE_WRITE(&glibc_rseq_offset, 0, false); - ATOMIC_4BYTE_WRITE(&glibc_rseq_size, 0, false); - } - SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); } void @@ -305,7 +229,7 @@ static void rseq_clear_tls_ptr(dcontext_t *dcontext) { ASSERT(rseq_tls_offset != 0); - byte *base = get_app_segment_base(LIB_SEG_TLS); + byte *base = get_segment_base(LIB_SEG_TLS); struct rseq *app_rseq = (struct rseq *)(base + rseq_tls_offset); /* We're directly writing this in the cache, so we do not bother with safe_read * or safe_write here either. We already cannot handle rseq adversarial cases. @@ -386,11 +310,6 @@ rseq_shared_fragment_flushtime_update(dcontext_t *dcontext) bool rseq_is_registered_for_current_thread(void) { - /* If glibc's rseq support is enabled, a struct rseq is already - * registered for all threads. - */ - if (glibc_rseq_size > 0) - return true; /* Unfortunately there's no way to query the current rseq struct. * For 64-bit we can pass a kernel address and look for EFAULT * vs EINVAL, but there is no kernel address for 32-bit. @@ -549,13 +468,7 @@ rseq_process_elf_sections(module_area_t *ma, bool at_map, * relocated and only need the offset if relocations have not yet been applied. */ ssize_t entry_offs = 0; - bool adjust_entry_offs = - at_map || (DYNAMO_OPTION(early_inject) && !dr_api_entry && !dynamo_started); - if (have_glibc_rseq) { - /* We want to set entry_offs if glibc has not been initialized yet. */ - adjust_entry_offs |= (glibc_rseq_offset == 0); - } - if (adjust_entry_offs) + if (at_map || (DYNAMO_OPTION(early_inject) && !dr_api_entry && !dynamo_started)) entry_offs = load_offs; for (i = 0; i < elf_hdr->e_shnum; i++) { #define RSEQ_PTR_ARRAY_SEC_NAME "__rseq_cs_ptr_array" @@ -714,11 +627,6 @@ rseq_process_module(module_area_t *ma, bool at_map) static int rseq_locate_tls_offset(void) { - /* If glibc's rseq support is enabled, we already know the offset. */ - if (glibc_rseq_size > 0) { - ASSERT(rseq_tls_offset > 0 && glibc_rseq_offset == rseq_tls_offset); - return rseq_tls_offset; - } /* We assume (and document) that the loader's static TLS is used, so every thread * has a consistent %fs:-offs address. Unfortunately, using a local copy of the * rseq code for our non-instrumented execution requires us to locate the app's @@ -827,12 +735,7 @@ rseq_locate_rseq_regions(void) d_r_mutex_unlock(&rseq_trigger_lock); return; } - rseq_check_glibc_support(); - /* If the app's glibc version supports rseq, then we should have already - * determined rseq_tls_offset, either by rseq_check_glibc_support or - * rseq_process_syscall (for non-attach cases). - */ - ASSERT(glibc_rseq_size == 0 || rseq_tls_offset > 0); + int offset = 0; if (rseq_tls_offset == 0) { /* Identify the TLS offset of this thread's struct rseq. */ @@ -865,9 +768,6 @@ rseq_locate_rseq_regions(void) void rseq_module_init(module_area_t *ma, bool at_map) { - /* If !rseq_enabled, the module will simply be processed when we enable - * rseq. - */ if (rseq_enabled) { rseq_process_module(ma, at_map); } From cd6d8c81b5c161dad4ac4547a2115f645bddab72 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Thu, 10 Nov 2022 12:22:50 -0500 Subject: [PATCH 26/36] Alternate method of detecting glibc rseq. --- core/unix/os.c | 5 +-- core/unix/rseq_linux.c | 92 ++++++++++++++++++++++++++++++------------ core/unix/rseq_linux.h | 2 +- 3 files changed, 70 insertions(+), 29 deletions(-) diff --git a/core/unix/os.c b/core/unix/os.c index ffb1f5649d6..715461a8d3d 100644 --- a/core/unix/os.c +++ b/core/unix/os.c @@ -9371,7 +9371,6 @@ post_system_call(dcontext_t *dcontext) /* Lazy rseq handling. */ if (success) { rseq_process_syscall(dcontext); - rseq_locate_rseq_regions(); } break; #endif @@ -10732,7 +10731,7 @@ os_take_over_all_unknown_threads(dcontext_t *dcontext) #ifdef LINUX /* Check this thread for rseq in between setup and start. */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); #endif /* Find tids for which we have no thread record, meaning they are not under @@ -10960,7 +10959,7 @@ os_thread_take_over(priv_mcontext_t *mc, kernel_sigset_t *sigset) * regions as rseq when the rseq syscall is never set up. */ if (rseq_is_registered_for_current_thread()) { - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); rseq_thread_attach(dcontext); } #endif diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index c0c691644e9..2a926ee9a2b 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -73,6 +73,10 @@ struct rseq { } __attribute__((aligned(4 * sizeof(uint64)))); #define RSEQ_FLAG_UNREGISTER 1 +#ifdef X86 +# define GLIBC_RSEQ_OFFSET 2464 +#endif + vm_area_vector_t *d_r_rseq_areas; DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, INIT_LOCK_FREE(rseq_trigger_lock)); @@ -163,7 +167,7 @@ d_r_rseq_init(void) rseq_cs_free _IF_DEBUG("rseq_cs table")); /* Enable rseq pre-attach for things like dr_prepopulate_cache(). */ if (rseq_is_registered_for_current_thread()) - rseq_locate_rseq_regions(); + rseq_locate_rseq_regions(false); } void @@ -229,7 +233,7 @@ static void rseq_clear_tls_ptr(dcontext_t *dcontext) { ASSERT(rseq_tls_offset != 0); - byte *base = get_segment_base(LIB_SEG_TLS); + byte *base = get_app_segment_base(LIB_SEG_TLS); struct rseq *app_rseq = (struct rseq *)(base + rseq_tls_offset); /* We're directly writing this in the cache, so we do not bother with safe_read * or safe_write here either. We already cannot handle rseq adversarial cases. @@ -545,7 +549,7 @@ rseq_process_elf_sections(module_area_t *ma, bool at_map, /* Returns whether successfully searched for rseq data (not whether found rseq data). */ static bool -rseq_process_module(module_area_t *ma, bool at_map) +rseq_process_module(module_area_t *ma, bool at_map, bool saw_glibc_rseq_reg) { bool res = false; ASSERT(is_elf_so_header(ma->start, ma->end - ma->start)); @@ -596,7 +600,11 @@ rseq_process_module(module_area_t *ma, bool at_map) goto rseq_process_module_cleanup; strtab = (char *)(str_map + sec_hdr[elf_hdr->e_shstrndx].sh_offset - offs); } - rseq_process_elf_sections(ma, at_map, sec_hdr, strtab, load_offs); + /* When saw_glibc_rseq_reg is set, we are still at glibc init, and ld has not + * relocated the executable yet. + */ + rseq_process_elf_sections(ma, at_map || saw_glibc_rseq_reg, sec_hdr, strtab, + load_offs); res = true; rseq_process_module_cleanup: if (str_size != 0) @@ -621,6 +629,31 @@ rseq_process_module(module_area_t *ma, bool at_map) return res; } +static bool +try_struct_rseq(void *try_addr) +{ + static const int RSEQ_RARE_SIGNATURE = 42; + int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), + RSEQ_FLAG_UNREGISTER, RSEQ_RARE_SIGNATURE); + LOG(GLOBAL, LOG_LOADER, 3, "Tried rseq @ " PFX " => %d\n", try_addr, res); + if (res == -EINVAL) /* Our struct != registered struct. */ + return false; + /* We expect -EPERM on a signature mismatch. On the small chance the app + * actually used 42 for its signature we'll have to re-register it. + */ + if (res == 0) { + int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), 0, + RSEQ_RARE_SIGNATURE); + ASSERT(res == 0); + res = -EPERM; + } + if (res == -EPERM) { + /* Found it! */ + return true; + } + return false; +} + /* If we did not observe the app invoke SYS_rseq (because we attached mid-run) * we must search for its TLS location. */ @@ -658,31 +691,27 @@ rseq_locate_tls_offset(void) * find the registered one. Our caller is not supposed to call here * until the app has registered the current thread. */ - static const int RSEQ_RARE_SIGNATURE = 42; - int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), - RSEQ_FLAG_UNREGISTER, RSEQ_RARE_SIGNATURE); - LOG(GLOBAL, LOG_LOADER, 3, "Tried rseq @ " PFX " => %d\n", try_addr, res); - if (res == -EINVAL) /* Our struct != registered struct. */ - continue; - /* We expect -EPERM on a signature mismatch. On the small chance the app - * actually used 42 for its signature we'll have to re-register it. - */ - if (res == 0) { - int res = dynamorio_syscall(SYS_rseq, 4, try_addr, sizeof(struct rseq), 0, - RSEQ_RARE_SIGNATURE); - ASSERT(res == 0); - res = -EPERM; - } - if (res == -EPERM) { - /* Found it! */ + if (try_struct_rseq(try_addr)) { LOG(GLOBAL, LOG_LOADER, 2, "Found struct rseq @ " PFX " for thread => %s:%s0x%x\n", try_addr, get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), i * alignment); offset = IF_X86_ELSE(-i * alignment, i * alignment); + break; } - break; } } +#ifdef X86 + if (offset == 0) { + if (try_struct_rseq(addr + GLIBC_RSEQ_OFFSET)) { + offset = GLIBC_RSEQ_OFFSET; + } + /* TODO i#5431: Sweep aligned positive offsets for the struct rseq instead of + * hard-coding the expected offset, since it may change in future. + */ + } +#else + /* TODO i#5431: Add support glibc 2.35+ on non-x86. */ +#endif return offset; } @@ -692,12 +721,19 @@ rseq_process_syscall(dcontext_t *dcontext) byte *seg_base = get_app_segment_base(LIB_SEG_TLS); byte *app_addr = (byte *)dcontext->sys_param0; bool constant_offset = false; +#ifdef X86 + bool first_rseq_reg = false; +#endif if (rseq_tls_offset == 0) { SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); int offset = app_addr - seg_base; /* To handle races here, we use an atomic_exchange. */ int prior = atomic_exchange_int(&rseq_tls_offset, offset); SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); +#ifdef X86 + if (prior == 0) + first_rseq_reg = true; +#endif constant_offset = (prior == 0 || prior == offset); LOG(GLOBAL, LOG_LOADER, 2, "Observed struct rseq @ " PFX " for thread => %s:%s0x%x\n", app_addr, @@ -711,6 +747,12 @@ rseq_process_syscall(dcontext_t *dcontext) "struct rseq is not always in static thread-local storage"); ASSERT_NOT_REACHED(); } + /* The struct rseq registered by glibc 2.35+ is inside struct pthread, which is + * at a positive offset from thread pointer, unlike the static TLS used by manual + * application registration. + */ + rseq_locate_rseq_regions( + IF_X86_ELSE(first_rseq_reg && rseq_tls_offset == GLIBC_RSEQ_OFFSET, false)); } /* Restartable sequence region identification. @@ -723,7 +765,7 @@ rseq_process_syscall(dcontext_t *dcontext) * the app has registered the current thread for rseq. */ void -rseq_locate_rseq_regions(void) +rseq_locate_rseq_regions(bool saw_glibc_rseq_reg) { if (rseq_enabled) return; @@ -759,7 +801,7 @@ rseq_locate_rseq_regions(void) module_iterator_t *iter = module_iterator_start(); while (module_iterator_hasnext(iter)) { module_area_t *ma = module_iterator_next(iter); - rseq_process_module(ma, false /*!at_map*/); + rseq_process_module(ma, false /*!at_map*/, saw_glibc_rseq_reg); } module_iterator_stop(iter); d_r_mutex_unlock(&rseq_trigger_lock); @@ -769,7 +811,7 @@ void rseq_module_init(module_area_t *ma, bool at_map) { if (rseq_enabled) { - rseq_process_module(ma, at_map); + rseq_process_module(ma, at_map, false); } } diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index 56a25aa4dd9..b7dd16d256b 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -58,7 +58,7 @@ bool rseq_is_registered_for_current_thread(void); void -rseq_locate_rseq_regions(void); +rseq_locate_rseq_regions(bool); void rseq_module_init(module_area_t *ma, bool at_map); From ea7b07a215799a13c566b6adc2a3513f98fb8058 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Thu, 10 Nov 2022 16:10:14 -0500 Subject: [PATCH 27/36] Add glibc 2.35+ support for aarch64, and prevent hardcoding rseq tls offset. --- core/unix/rseq_linux.c | 45 ++++++++++++---------------------------- suite/tests/linux/rseq.c | 11 +++++++++- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 2a926ee9a2b..bb59a3f0f84 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -73,10 +73,6 @@ struct rseq { } __attribute__((aligned(4 * sizeof(uint64)))); #define RSEQ_FLAG_UNREGISTER 1 -#ifdef X86 -# define GLIBC_RSEQ_OFFSET 2464 -#endif - vm_area_vector_t *d_r_rseq_areas; DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, INIT_LOCK_FREE(rseq_trigger_lock)); @@ -682,11 +678,12 @@ rseq_locate_tls_offset(void) /* For x86, static TLS is at a negative offset from the app library segment * base, while for aarchxx it is positive. */ - for (i = 0; IF_X86_ELSE(addr - i * alignment >= seg_bottom, - addr + i * alignment < seg_bottom + seg_size); - i++) { - byte *try_addr = IF_X86_ELSE(addr - i * alignment, addr + i * alignment); - ASSERT(try_addr >= seg_bottom); /* For loop guarantees this. */ + ASSERT(seg_bottom <= addr && addr < seg_bottom + seg_size); + for (i = (seg_bottom - addr) / alignment; + addr + i * alignment < seg_bottom + seg_size; ++i) { + byte *try_addr = addr + i * alignment; + ASSERT(seg_bottom <= try_addr && + try_addr < seg_bottom + seg_size); /* For loop guarantees this. */ /* Our strategy is to check all of the aligned static TLS addresses to * find the registered one. Our caller is not supposed to call here * until the app has registered the current thread. @@ -695,23 +692,11 @@ rseq_locate_tls_offset(void) LOG(GLOBAL, LOG_LOADER, 2, "Found struct rseq @ " PFX " for thread => %s:%s0x%x\n", try_addr, get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), i * alignment); - offset = IF_X86_ELSE(-i * alignment, i * alignment); + offset = i * alignment; break; } } } -#ifdef X86 - if (offset == 0) { - if (try_struct_rseq(addr + GLIBC_RSEQ_OFFSET)) { - offset = GLIBC_RSEQ_OFFSET; - } - /* TODO i#5431: Sweep aligned positive offsets for the struct rseq instead of - * hard-coding the expected offset, since it may change in future. - */ - } -#else - /* TODO i#5431: Add support glibc 2.35+ on non-x86. */ -#endif return offset; } @@ -721,19 +706,15 @@ rseq_process_syscall(dcontext_t *dcontext) byte *seg_base = get_app_segment_base(LIB_SEG_TLS); byte *app_addr = (byte *)dcontext->sys_param0; bool constant_offset = false; -#ifdef X86 - bool first_rseq_reg = false; -#endif + bool first_rseq_registration = false; if (rseq_tls_offset == 0) { SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT); int offset = app_addr - seg_base; /* To handle races here, we use an atomic_exchange. */ int prior = atomic_exchange_int(&rseq_tls_offset, offset); SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); -#ifdef X86 if (prior == 0) - first_rseq_reg = true; -#endif + first_rseq_registration = true; constant_offset = (prior == 0 || prior == offset); LOG(GLOBAL, LOG_LOADER, 2, "Observed struct rseq @ " PFX " for thread => %s:%s0x%x\n", app_addr, @@ -748,11 +729,11 @@ rseq_process_syscall(dcontext_t *dcontext) ASSERT_NOT_REACHED(); } /* The struct rseq registered by glibc 2.35+ is inside struct pthread, which is - * at a positive offset from thread pointer, unlike the static TLS used by manual - * application registration. + * at a positive offset from thread pointer on x86 and negative offset on AArch64, + * both unlike the static TLS used by manual app registration. */ - rseq_locate_rseq_regions( - IF_X86_ELSE(first_rseq_reg && rseq_tls_offset == GLIBC_RSEQ_OFFSET, false)); + rseq_locate_rseq_regions(first_rseq_registration && + IF_X86_ELSE(rseq_tls_offset > 0, rseq_tls_offset < 0)); } /* Restartable sequence region identification. diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index 16fd1e2321d..4c956566204 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -117,8 +117,17 @@ register_rseq() { #ifdef GLIBC_RSEQ if (__rseq_size > 0) { + /* Our glibc rseq handling in rseq_linux.c does not depend on the exact + * offset, but it does depend on it being in struct pthread, therefore + * the sign of __rseq_offset. Nevertheless, we check for the exact + * offset here so that we know if glibc changes how it handles rseq. + */ +# ifdef X86 + assert(__rseq_offset == 2464); +# else + assert(__rseq_offset == -32); +# endif /* Already registered by glibc. Verify that it's there. */ - assert(__rseq_offset > 0); struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, 0); assert(res == -1 && (errno == EPERM || errno == ENOSYS)); From 8a0beddb62223285a411fb418bb61589c45093c8 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Thu, 10 Nov 2022 16:24:36 -0500 Subject: [PATCH 28/36] Some cleanup. --- core/unix/rseq_linux.c | 8 ++++++-- suite/tests/linux/rseq.c | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index bb59a3f0f84..018ddb97d02 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -675,8 +675,12 @@ rseq_locate_tls_offset(void) /* struct rseq_cs is aligned to 32. */ int alignment = __alignof(struct rseq_cs); int i; - /* For x86, static TLS is at a negative offset from the app library segment - * base, while for aarchxx it is positive. + /* When rseq support is enabled in glibc 2.35+, the glibc-registered struct rseq + * is present in the struct pthread, which is at a positive offset from the + * app library segment base on x86, and negative on aarchxx. However, in the + * absence of rseq support from glibc, the app manually registers its own + * struct rseq which is present in static TLS, which is at a negative offset + * from the app library segment base on x86, and positive on aarchxx. */ ASSERT(seg_bottom <= addr && addr < seg_bottom + seg_size); for (i = (seg_bottom - addr) / alignment; diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index 4c956566204..f36e1768a9a 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -1025,8 +1025,7 @@ int main() { intercept_signal(SIGILL, signal_handler, false); - bool res = register_rseq(); - if (res) { + if (register_rseq()) { #ifdef RSEQ_TEST_ATTACH /* Set -offline to avoid trying to open a pipe to a missing reader. */ if (setenv("DYNAMORIO_OPTIONS", "-stderr_mask 0xc -client_lib ';;-offline'", From 7c0ae884f3bedab880e6515e0e4463ca67a0e6ea Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Thu, 10 Nov 2022 22:34:49 -0500 Subject: [PATCH 29/36] Some cleanup. --- core/unix/rseq_linux.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 018ddb97d02..20e1bb0353c 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -688,14 +688,15 @@ rseq_locate_tls_offset(void) byte *try_addr = addr + i * alignment; ASSERT(seg_bottom <= try_addr && try_addr < seg_bottom + seg_size); /* For loop guarantees this. */ - /* Our strategy is to check all of the aligned static TLS addresses to - * find the registered one. Our caller is not supposed to call here - * until the app has registered the current thread. + /* Our strategy is to check all of the aligned addresses to find the + * registered one. Our caller is not supposed to call here until the app + * has registered the current thread (either manually or using glibc). */ if (try_struct_rseq(try_addr)) { LOG(GLOBAL, LOG_LOADER, 2, "Found struct rseq @ " PFX " for thread => %s:%s0x%x\n", try_addr, - get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), i * alignment); + get_register_name(LIB_SEG_TLS), (i < 0 ? "-" : ""), + abs(i) * alignment); offset = i * alignment; break; } From 77d30f07b63f8cc02fe04e4233740434c47024f8 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Thu, 10 Nov 2022 23:33:21 -0500 Subject: [PATCH 30/36] Fix log. --- core/unix/rseq_linux.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 20e1bb0353c..320743d8586 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -723,8 +723,8 @@ rseq_process_syscall(dcontext_t *dcontext) constant_offset = (prior == 0 || prior == offset); LOG(GLOBAL, LOG_LOADER, 2, "Observed struct rseq @ " PFX " for thread => %s:%s0x%x\n", app_addr, - get_register_name(LIB_SEG_TLS), IF_X86_ELSE("-", ""), - IF_X86_ELSE(-rseq_tls_offset, rseq_tls_offset)); + get_register_name(LIB_SEG_TLS), (rseq_tls_offset < 0 ? "-" : ""), + abs(rseq_tls_offset)); } else constant_offset = (seg_base + rseq_tls_offset == app_addr); if (!constant_offset) { From 320713db5d3581808aa273ca572478097f10b0a3 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 11 Nov 2022 00:03:46 -0500 Subject: [PATCH 31/36] Clarify registered rseq check. --- suite/tests/linux/rseq.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index f36e1768a9a..b4768877e88 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -129,8 +129,13 @@ register_rseq() # endif /* Already registered by glibc. Verify that it's there. */ struct rseq *reg_rseq = __builtin_thread_pointer() + __rseq_offset; - int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, 0); - assert(res == -1 && (errno == EPERM || errno == ENOSYS)); + /* Glibc uses RSEQ_SIG as the signature. The following will return + * EBUSY if reg_rseq is the registered struct rseq and EINVAL otherwise. + * FTR: if we use some other value as signature (like zero), this + * becomes an EPERM vs EINVAL check. + */ + int res = syscall(SYS_rseq, reg_rseq, sizeof(*reg_rseq), 0, RSEQ_SIG); + assert(res == -1 && (errno == EBUSY || errno == ENOSYS)); } else { #endif rseq_tls.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; From 6bfd645dcabb5880fef30a2c3f9a4339f56256f7 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 11 Nov 2022 16:03:37 -0500 Subject: [PATCH 32/36] Check expected glibc rseq offset first. --- core/unix/rseq_linux.c | 25 +++++++++++++++++++++++++ suite/tests/linux/rseq.c | 8 ++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 320743d8586..1cd757e2fa5 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -78,6 +78,18 @@ DECLARE_CXTSWPROT_VAR(static mutex_t rseq_trigger_lock, INIT_LOCK_FREE(rseq_trigger_lock)); static volatile bool rseq_enabled; +/* The struct rseq registered by glibc is present in the struct pthread. + * As of glibc 2.35, it is present at the following offset from the app + * lib seg base. We check these offsets first and then fall back to a + * wider search. The linux.rseq test helps detect changes in these + * offsets in future glibc versions. + */ +#ifdef X86 +# define GLIBC_RSEQ_OFFSET 2464 +#else +# define GLIBC_RSEQ_OFFSET -32 +#endif + /* We require all threads to use the same TLS offset to point at struct rseq. */ static int rseq_tls_offset; @@ -667,6 +679,19 @@ rseq_locate_tls_offset(void) */ int offset = 0; byte *addr = get_app_segment_base(LIB_SEG_TLS); + if (addr > 0) { + byte *try_glibc_addr = addr + GLIBC_RSEQ_OFFSET; + if (try_struct_rseq(try_glibc_addr)) { + LOG(GLOBAL, LOG_LOADER, 2, + "Found glibc struct rseq @ " PFX " for thread => %s:%s0x%x\n", + try_glibc_addr, get_register_name(LIB_SEG_TLS), + (GLIBC_RSEQ_OFFSET < 0 ? "-" : ""), abs(GLIBC_RSEQ_OFFSET)); + return GLIBC_RSEQ_OFFSET; + } + } + /* Either the app's glibc does not have rseq support (old glibc or disabled by app) + * or the offset of glibc's struct rseq has changed. We do a wider search now. + */ byte *seg_bottom; size_t seg_size; if (addr > 0 && get_memory_info(addr, &seg_bottom, &seg_size, NULL)) { diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index b4768877e88..8bccfd06ee8 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -117,10 +117,10 @@ register_rseq() { #ifdef GLIBC_RSEQ if (__rseq_size > 0) { - /* Our glibc rseq handling in rseq_linux.c does not depend on the exact - * offset, but it does depend on it being in struct pthread, therefore - * the sign of __rseq_offset. Nevertheless, we check for the exact - * offset here so that we know if glibc changes how it handles rseq. + /* Our glibc rseq handling in rseq_linux.c checks the following offset + * first for the glibc-registered struct rseq. Though we do have a + * fallback that does a wider search, it would be good to keep the + * expected offset in sync with glibc changes. */ # ifdef X86 assert(__rseq_offset == 2464); From c092a1a56056b974df7143a3ca850f2114c61123 Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 11 Nov 2022 17:22:57 -0500 Subject: [PATCH 33/36] Review comments. --- core/unix/rseq_linux.c | 14 ++++++++++---- core/unix/rseq_linux.h | 2 +- suite/tests/linux/rseq.c | 4 ++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 1cd757e2fa5..4d03a10f40d 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -51,6 +51,8 @@ #include "include/syscall.h" #include +#define ABS(x) ((x) < 0 ? -(x) : (x)) + /* The linux/rseq.h header made a source-breaking change in torvalds/linux@bfdf4e6 * which broke our build. To avoid future issues we use our own definitions. * Binary breakage is unlikely without long periods of deprecation so this is @@ -85,7 +87,11 @@ static volatile bool rseq_enabled; * offsets in future glibc versions. */ #ifdef X86 -# define GLIBC_RSEQ_OFFSET 2464 +# ifdef X64 +# define GLIBC_RSEQ_OFFSET 2464 +# else +# define GLIBC_RSEQ_OFFSET 1312 +# endif #else # define GLIBC_RSEQ_OFFSET -32 #endif @@ -685,7 +691,7 @@ rseq_locate_tls_offset(void) LOG(GLOBAL, LOG_LOADER, 2, "Found glibc struct rseq @ " PFX " for thread => %s:%s0x%x\n", try_glibc_addr, get_register_name(LIB_SEG_TLS), - (GLIBC_RSEQ_OFFSET < 0 ? "-" : ""), abs(GLIBC_RSEQ_OFFSET)); + (GLIBC_RSEQ_OFFSET < 0 ? "-" : ""), ABS(GLIBC_RSEQ_OFFSET)); return GLIBC_RSEQ_OFFSET; } } @@ -721,7 +727,7 @@ rseq_locate_tls_offset(void) LOG(GLOBAL, LOG_LOADER, 2, "Found struct rseq @ " PFX " for thread => %s:%s0x%x\n", try_addr, get_register_name(LIB_SEG_TLS), (i < 0 ? "-" : ""), - abs(i) * alignment); + ABS(i) * alignment); offset = i * alignment; break; } @@ -749,7 +755,7 @@ rseq_process_syscall(dcontext_t *dcontext) LOG(GLOBAL, LOG_LOADER, 2, "Observed struct rseq @ " PFX " for thread => %s:%s0x%x\n", app_addr, get_register_name(LIB_SEG_TLS), (rseq_tls_offset < 0 ? "-" : ""), - abs(rseq_tls_offset)); + ABS(rseq_tls_offset)); } else constant_offset = (seg_base + rseq_tls_offset == app_addr); if (!constant_offset) { diff --git a/core/unix/rseq_linux.h b/core/unix/rseq_linux.h index b7dd16d256b..e28bfb7b9c1 100644 --- a/core/unix/rseq_linux.h +++ b/core/unix/rseq_linux.h @@ -58,7 +58,7 @@ bool rseq_is_registered_for_current_thread(void); void -rseq_locate_rseq_regions(bool); +rseq_locate_rseq_regions(bool saw_glibc_rseq_reg); void rseq_module_init(module_area_t *ma, bool at_map); diff --git a/suite/tests/linux/rseq.c b/suite/tests/linux/rseq.c index 8bccfd06ee8..e6e5650895b 100644 --- a/suite/tests/linux/rseq.c +++ b/suite/tests/linux/rseq.c @@ -123,7 +123,11 @@ register_rseq() * expected offset in sync with glibc changes. */ # ifdef X86 +# ifdef X64 assert(__rseq_offset == 2464); +# else + assert(__rseq_offset == 1312); +# endif # else assert(__rseq_offset == -32); # endif From 9752a29e70581455d23e6762b4985938b331ec9e Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 11 Nov 2022 17:31:21 -0500 Subject: [PATCH 34/36] Add comment about AArch32. --- core/unix/rseq_linux.c | 1 + 1 file changed, 1 insertion(+) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 4d03a10f40d..ea8ca95615c 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -93,6 +93,7 @@ static volatile bool rseq_enabled; # define GLIBC_RSEQ_OFFSET 1312 # endif #else +/* This was verified on AArch64, but not on AArch32. */ # define GLIBC_RSEQ_OFFSET -32 #endif From 9489250082cc4c86c0167555452a0dadcaebbc2c Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Fri, 11 Nov 2022 17:47:48 -0500 Subject: [PATCH 35/36] Update AArch32 comment. --- core/unix/rseq_linux.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index ea8ca95615c..2aaf5e78645 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -93,7 +93,10 @@ static volatile bool rseq_enabled; # define GLIBC_RSEQ_OFFSET 1312 # endif #else -/* This was verified on AArch64, but not on AArch32. */ +/* This was verified on AArch64, but not on AArch32. + * XXX: To improve struct rseq offset detection on AArch32, find the offset + * on an AArch32 machine running glibc 2.35+ and add here. + */ # define GLIBC_RSEQ_OFFSET -32 #endif From 949bd92423f876e2152a01846b43399b268ade0a Mon Sep 17 00:00:00 2001 From: Abhinav Anil Sharma Date: Sun, 13 Nov 2022 10:10:13 -0500 Subject: [PATCH 36/36] Improve logs. --- core/unix/rseq_linux.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/unix/rseq_linux.c b/core/unix/rseq_linux.c index 2aaf5e78645..3537a5396bb 100644 --- a/core/unix/rseq_linux.c +++ b/core/unix/rseq_linux.c @@ -757,15 +757,15 @@ rseq_process_syscall(dcontext_t *dcontext) first_rseq_registration = true; constant_offset = (prior == 0 || prior == offset); LOG(GLOBAL, LOG_LOADER, 2, - "Observed struct rseq @ " PFX " for thread => %s:%s0x%x\n", app_addr, - get_register_name(LIB_SEG_TLS), (rseq_tls_offset < 0 ? "-" : ""), + "Observed struct rseq at syscall @ " PFX " for thread => %s:%s0x%x\n", + app_addr, get_register_name(LIB_SEG_TLS), (rseq_tls_offset < 0 ? "-" : ""), ABS(rseq_tls_offset)); } else constant_offset = (seg_base + rseq_tls_offset == app_addr); if (!constant_offset) { - REPORT_FATAL_ERROR_AND_EXIT( - RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), get_application_pid(), - "struct rseq is not always in static thread-local storage"); + REPORT_FATAL_ERROR_AND_EXIT(RSEQ_BEHAVIOR_UNSUPPORTED, 3, get_application_name(), + get_application_pid(), + "struct rseq is not always at the same offset"); ASSERT_NOT_REACHED(); } /* The struct rseq registered by glibc 2.35+ is inside struct pthread, which is