i#3544 RV64: Fix signal mask restoring after detection of RVV (#6890)

We detect the RISC-V vector extension using SIGILL (PR #6836). However,
using `dr_longjmp` to exit the signal handler would miss to restore the
signal mask, which is operated by the OS after exiting the signal
handle. Therefore, we need to manually save and restore the signal mask.

Issue: #3544

core/unix/signal.c

@@ -456,7 +456,7 @@ setitimer_syscall(int which, struct itimerval *val, struct itimerval *old)
     return dynamorio_syscall(SYS_setitimer, 3, which, val, old);
 }
 
-static inline int
+int
 sigprocmask_syscall(int how, kernel_sigset_t *set, kernel_sigset_t *oset,
                     size_t sigsetsize)
 {

core/unix/signal_linux_riscv64.c

@@ -148,10 +148,19 @@ sigill_detected(void *func)
     set_handler_sigact(&act, SIGILL, (handler_t)catch_sigill);
     sigaction_syscall(SIGILL, &act, &old_act);
 
+    /* We use dr_longjmp to exit the SIGILL handler, which skips the signal mask restoring
+     * of OS. Manually save and restore the signal mask here.
+     * XXX: Add dr_longjmp_sigmask() to make this easier?
+     */
+    kernel_sigset_t oset;
+    sigprocmask_syscall(SIG_SETMASK, NULL, &oset, sizeof(oset));
+
     if (dr_setjmp(&jmpbuf) == 0) {
         ((void (*)(void))func)();
     }
 
+    sigprocmask_syscall(SIG_SETMASK, &oset, NULL, sizeof(oset));
+
     sigaction_syscall(SIGILL, &old_act, NULL);
 
     return sigill_caught;

core/unix/signal_private.h

@@ -648,6 +648,10 @@ sigaction_syscall(int sig, kernel_sigaction_t *act, kernel_sigaction_t *oact);
 void
 set_handler_sigact(kernel_sigaction_t *act, int sig, handler_t handler);
 
+int
+sigprocmask_syscall(int how, kernel_sigset_t *set, kernel_sigset_t *oset,
+                    size_t sigsetsize);
+
 /***************************************************************************
  * OS-SPECIFIC ROUTINES (in signal_<os>.c)
  */