i#5505 PT tracing: Add burst PT test with interrupted futex (#7032)

Adds a test where one of the threads is waiting on a futex when detach
occurs. PT traces for such futex syscalls have been observed to fail in
libipt decode. We also do not want such PT traces because they do not
represent real app behavior, as the syscall was interrupted by DR's
detach signal. #7027 added logic to skip them from the written trace.

This PR adds a regression test. Unfortunately this test still does not
reproduce the original libipt decode issue that was seen on a large app.
Most errors seen were on a modified kernel and only a few on a regular
futex. But it is still useful to add this test that ensures that the
thread-final interrupted syscall is skipped.

This test also uncovers a possible transparency violation seen in the
behavior of an interrupted-and-restarted futex call, where the blocked
thread doesn't remember that it was supposed to wait on a different
futex specified by a later FUTEX_CMP_REQUEUE call than the one specified
by it in the original futex syscall.

Since the new test requires Intel-PT, verified that it passes by running
it manually locally:

```
The following tests passed:
	code_api|tool.drcacheoff.burst_syscall_pt_SUDO

The following tests passed:
	code_api|tool.drcacheoff.kernel.simple_SUDO
	code_api|tool.drcacheoff.kernel.opcode-mix_SUDO
	code_api|tool.drcacheoff.kernel.syscall-mix_SUDO
	code_api|tool.drcacheoff.kernel.invariant-checker_SUDO

```

Issue: #5505
Issue: #7034

clients/drcachesim/CMakeLists.txt

@@ -1086,6 +1086,17 @@ if (BUILD_TESTS)
       use_DynamoRIO_drmemtrace_tracer(tool.drcacheoff.burst_syscall_inject)
     endif ()
 
+    if (LINUX AND BUILD_PT_POST_PROCESSOR AND BUILD_PT_TRACER)
+      add_executable(tool.drcacheoff.burst_syscall_pt_SUDO tests/burst_syscall_pt.cpp)
+      configure_DynamoRIO_static(tool.drcacheoff.burst_syscall_pt_SUDO)
+      use_DynamoRIO_static_client(tool.drcacheoff.burst_syscall_pt_SUDO drmemtrace_static)
+      target_link_libraries(tool.drcacheoff.burst_syscall_pt_SUDO drmemtrace_raw2trace
+          drmemtrace_analyzer test_helpers drmemtrace_basic_counts)
+      add_win32_flags(tool.drcacheoff.burst_syscall_pt_SUDO)
+      use_DynamoRIO_drmemtrace_tracer(tool.drcacheoff.burst_syscall_pt_SUDO)
+      link_with_pthread(tool.drcacheoff.burst_syscall_pt_SUDO)
+    endif ()
+
     if (UNIX)
       if (X86 AND NOT APPLE) # This test is x86-specific.
         # uses ptrace and looks for linux-specific syscalls

clients/drcachesim/tests/burst_syscall_pt.cpp

@@ -0,0 +1,347 @@
+/* **********************************************************
+ * Copyright (c) 2016-2024 Google, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Google, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+// This application links in drmemtrace_static and acquires a trace during
+// a "burst" of execution that includes some system call traces collected
+// using Intel-PT.
+
+// This is set globally in CMake for other tests so easier to undef here.
+#undef DR_REG_ENUM_COMPATIBILITY
+
+#include "analyzer.h"
+#include "tools/basic_counts.h"
+#include "dr_api.h"
+#include "drmemtrace/drmemtrace.h"
+#include "drmemtrace/raw2trace.h"
+#include "mock_reader.h"
+#include "raw2trace_directory.h"
+#include "scheduler.h"
+
+#include <cassert>
+#include <unistd.h>
+#include <fstream>
+#include <iostream>
+#include <linux/futex.h>
+#include <pthread.h>
+#include <string>
+#include <sys/syscall.h>
+
+namespace dynamorio {
+namespace drmemtrace {
+
+#define FATAL_ERROR(msg, ...)                               \
+    do {                                                    \
+        fprintf(stderr, "ERROR: " msg "\n", ##__VA_ARGS__); \
+        fflush(stderr);                                     \
+        exit(1);                                            \
+    } while (0)
+
+/* The futex the child waits at initially. */
+static uint32_t futex_var = 0xf00d;
+/* The futex the child is transferred to. */
+static uint32_t futex_var_other = 0x8bad;
+
+static void *
+child_futex_wait(void *)
+{
+    long res = syscall(SYS_futex, &futex_var, FUTEX_WAIT, /*#val=*/0xf00d,
+                       /*timeout=*/nullptr, /*uaddr2=*/nullptr, /*val3=*/0);
+    assert(res == 0);
+    std::cerr << "Child released from futex\n";
+    return NULL;
+}
+
+static void
+parent_futex_wake()
+{
+    /* The child would be waiting at the other futex by now.
+     * i#7034: Note that the child thread undergoes detach while it is waiting
+     * on futex_var_other. There is a bug at this point due to a possible
+     * transparency violation in DR. When the child thread restarts futex after
+     * being interrupted by DR's detach signal, it is found to resume
+     * waiting at the original futex_var instead of futex_var_other.
+     * If we modify this app to do detach after parent_futex_wake returns, then
+     * the child is found to be waiting at futex_var_other as expected.
+     */
+    uint32_t *child_waiting_at_futex = &futex_var;
+    long res = syscall(SYS_futex, child_waiting_at_futex, FUTEX_WAKE, /*#wakeup=*/1,
+                       /*timeout=*/nullptr, /*uaddr2=*/nullptr, /*val3=*/0);
+    assert(res == 1);
+}
+
+static void
+parent_futex_reque()
+{
+    long res;
+    do {
+        /* Repeat until the child is surely waiting at the futex. We'll know this
+         * when the following call returns a 1, which means the child was
+         * transferred to futex_var_other. This is to ensure that the child thread
+         * is inside the futex syscall when DR detaches.
+         */
+        res = syscall(SYS_futex, &futex_var, FUTEX_CMP_REQUEUE, /*#wakeup_max=*/0,
+                      /*#requeue_max=*/1, /*uaddr2=*/&futex_var_other, /*val3=*/0xf00d);
+        assert(res == 0 || res == 1);
+    } while (res == 0);
+}
+
+static int
+do_some_syscalls()
+{
+    getpid();
+    gettid();
+    return 1;
+}
+
+static std::string
+postprocess(void *dr_context)
+{
+    std::cerr << "Post-processing the trace\n";
+    // Get path to write the final trace to.
+    const char *raw_dir;
+    drmemtrace_status_t mem_res = drmemtrace_get_output_path(&raw_dir);
+    assert(mem_res == DRMEMTRACE_SUCCESS);
+    std::string outdir = std::string(raw_dir) + DIRSEP + "post_processed";
+
+    const char *kcore_path;
+    drmemtrace_status_t kcore_res = drmemtrace_get_kcore_path(&kcore_path);
+    assert(kcore_res == DRMEMTRACE_SUCCESS);
+
+    raw2trace_directory_t dir;
+    if (!dr_create_dir(outdir.c_str()))
+        FATAL_ERROR("Failed to create output dir.");
+    std::string dir_err = dir.initialize(raw_dir, outdir, DEFAULT_TRACE_COMPRESSION_TYPE,
+                                         /*syscall_template_file=*/"");
+    assert(dir_err.empty());
+    raw2trace_t raw2trace(dir.modfile_bytes_, dir.in_files_, dir.out_files_,
+                          dir.out_archives_, dir.encoding_file_,
+                          dir.serial_schedule_file_, dir.cpu_schedule_file_, dr_context,
+                          /*verbosity=*/0, /*worker_count=*/-1,
+                          /*alt_module_dir=*/"",
+                          /*chunk_instr_count=*/10 * 1000 * 1000, dir.in_kfiles_map_,
+                          dir.kcoredir_, /*kallsyms_path=*/"",
+                          /*syscall_template_file=*/nullptr,
+                          // We want to fail if any error is encountered.
+                          /*pt2ir_best_effort=*/false);
+    std::string error = raw2trace.do_conversion();
+    if (!error.empty())
+        FATAL_ERROR("raw2trace failed: %s\n", error.c_str());
+    uint64 decoded_syscall_count =
+        raw2trace.get_statistic(RAW2TRACE_STAT_SYSCALL_TRACES_CONVERTED);
+    // We should see atleast the getpid, gettid, and futex syscalls made by the parent.
+    if (decoded_syscall_count <= 2) {
+        std::cerr << "Incorrect decoded syscall count (found: " << decoded_syscall_count
+                  << " vs expected > 2)\n";
+    }
+    return outdir;
+}
+
+// Trace analysis tool that allows us to verify properties of the generated PT trace.
+class pt_analysis_tool_t : public analysis_tool_t {
+public:
+    pt_analysis_tool_t()
+    {
+    }
+    bool
+    process_memref(const memref_t &memref) override
+    {
+        FATAL_ERROR("Expected to use sharded mode");
+        return true;
+    }
+    bool
+    parallel_shard_supported() override
+    {
+        return true;
+    }
+    void *
+    parallel_shard_init(int shard_index, void *worker_data) override
+    {
+        auto per_shard = new per_shard_t;
+        return reinterpret_cast<void *>(per_shard);
+    }
+    bool
+    parallel_shard_exit(void *shard_data) override
+    {
+        std::lock_guard<std::mutex> guard(shard_exit_mutex_);
+        per_shard_t *shard = reinterpret_cast<per_shard_t *>(shard_data);
+        if (shard->syscall_count == 0)
+            return true;
+        // In case the child has just the one futex syscall which was skipped
+        // from the trace.
+        if (shard->syscall_count > 1 && !shard->any_syscall_had_trace) {
+            std::cerr << "No syscall had a trace\n";
+        }
+        if (shard->prev_was_futex_marker && !shard->prev_syscall_had_trace) {
+            found_final_futex_without_trace_ = true;
+        }
+        if (shard->kernel_instr_count > 0) {
+            found_some_kernel_instrs_ = true;
+        }
+        return true;
+    }
+    bool
+    parallel_shard_memref(void *shard_data, const memref_t &memref) override
+    {
+        per_shard_t *shard = reinterpret_cast<per_shard_t *>(shard_data);
+        if (memref.marker.type == TRACE_TYPE_MARKER) {
+            switch (memref.marker.marker_type) {
+            case TRACE_MARKER_TYPE_SYSCALL_TRACE_START:
+                shard->in_syscall_trace = true;
+                break;
+            case TRACE_MARKER_TYPE_SYSCALL_TRACE_END:
+                shard->in_syscall_trace = false;
+                shard->prev_syscall_had_trace = true;
+                shard->any_syscall_had_trace = true;
+                break;
+            case TRACE_MARKER_TYPE_SYSCALL:
+                ++shard->syscall_count;
+                shard->prev_syscall_had_trace = false;
+                if (memref.marker.marker_value == SYS_futex) {
+                    shard->prev_was_futex_marker = true;
+                }
+                break;
+            }
+        }
+        if (!type_is_instr(memref.data.type))
+            return true;
+        if (shard->in_syscall_trace) {
+            ++shard->kernel_instr_count;
+            return true;
+        }
+        shard->prev_was_futex_marker = false;
+        shard->prev_syscall_had_trace = false;
+        return true;
+    }
+    bool
+    print_results() override
+    {
+        if (!found_final_futex_without_trace_) {
+            std::cerr
+                << "Did not find any thread trace with final futex without PT trace\n";
+        } else {
+            std::cerr << "Found matching signature in a thread\n";
+        }
+        if (!found_some_kernel_instrs_) {
+            std::cerr << "Did not find any kernel instrs\n";
+        }
+        return true;
+    }
+
+private:
+    // Data tracked per shard.
+    struct per_shard_t {
+        bool prev_was_futex_marker = false;
+        bool prev_syscall_had_trace = false;
+        bool any_syscall_had_trace = false;
+        int syscall_count = 0;
+        bool in_syscall_trace = false;
+        int kernel_instr_count = 0;
+    };
+
+    bool found_final_futex_without_trace_ = false;
+    bool found_some_kernel_instrs_ = false;
+    std::mutex shard_exit_mutex_;
+};
+
+static bool
+run_pt_analysis(const std::string &trace_dir)
+{
+    auto pt_analysis_tool = std::unique_ptr<pt_analysis_tool_t>(new pt_analysis_tool_t());
+    std::vector<analysis_tool_t *> tools;
+    tools.push_back(pt_analysis_tool.get());
+    analyzer_t analyzer(trace_dir, &tools[0], static_cast<int>(tools.size()));
+    if (!analyzer) {
+        FATAL_ERROR("failed to initialize analyzer: %s",
+                    analyzer.get_error_string().c_str());
+    }
+    if (!analyzer.run()) {
+        FATAL_ERROR("failed to run analyzer: %s", analyzer.get_error_string().c_str());
+    }
+    if (!analyzer.print_stats()) {
+        FATAL_ERROR("failed to print stats: %s", analyzer.get_error_string().c_str());
+    }
+    return true;
+}
+
+static void
+gather_trace()
+{
+    if (setenv("DYNAMORIO_OPTIONS",
+               "-stderr_mask 0xc -client_lib ';;-offline -enable_kernel_tracing",
+               1 /*override*/) != 0)
+        std::cerr << "failed to set env var!\n";
+    dr_app_setup();
+    assert(!dr_app_running_under_dynamorio());
+    dr_app_start();
+
+    pthread_t child_thread;
+    int res = pthread_create(&child_thread, NULL, child_futex_wait, NULL);
+    assert(res == 0);
+
+    /* Ensure that the child is waiting at a futex. */
+    parent_futex_reque();
+
+    do_some_syscalls();
+
+    dr_app_stop_and_cleanup();
+
+    /* Wake up the child finally. */
+    parent_futex_wake();
+
+    pthread_join(child_thread, NULL);
+
+    return;
+}
+
+static int
+test_pt_trace(void *dr_context)
+{
+    std::string trace_dir = postprocess(dr_context);
+    if (!run_pt_analysis(trace_dir))
+        return 1;
+    return 0;
+}
+
+int
+test_main(int argc, const char *argv[])
+{
+    gather_trace();
+    void *dr_context = dr_standalone_init();
+    if (test_pt_trace(dr_context)) {
+        return 1;
+    }
+    dr_standalone_exit();
+    return 0;
+}
+
+} // namespace drmemtrace
+} // namespace dynamorio

clients/drcachesim/tests/offline-burst_syscall_pt_SUDO.templatex

@@ -0,0 +1,9 @@
+ERROR: PT tracing for the last syscall .* of thread .* was found active at detach.
+Child released from futex
+Post-processing the trace
+Found matching signature in a thread
+Syscall mix tool results:
+  syscall count : syscall_num
+.*
+ syscall trace count : syscall_num
+.*

clients/drcachesim/tracer/tracer.cpp

@@ -1868,7 +1868,7 @@ event_thread_exit(void *drcontext)
             int cur_recording_sysnum = data->syscall_pt_trace.get_cur_recording_sysnum();
             if (cur_recording_sysnum != INVALID_SYSNUM) {
                 NOTIFY(0,
-                       "ERROR: PT tracing for the last syscall %d of thread T%d was  "
+                       "ERROR: PT tracing for the last syscall %d of thread T%d was "
                        "found active at detach.\n",
                        cur_recording_sysnum, dr_get_thread_id(drcontext));
                 // Ignore return value and try to continue in release build.