i#7113 decode cache: Add analyzer library for decode_cache_t

clients/drcachesim/CMakeLists.txt

@@ -161,19 +161,25 @@ add_exported_library(drmemtrace_reuse_distance STATIC tools/reuse_distance.cpp)
 add_exported_library(drmemtrace_histogram STATIC tools/histogram.cpp)
 add_exported_library(drmemtrace_reuse_time STATIC tools/reuse_time.cpp)
 add_exported_library(drmemtrace_basic_counts STATIC tools/basic_counts.cpp)
-add_exported_library(drmemtrace_opcode_mix STATIC
-                     tools/opcode_mix.cpp tracer/raw2trace_shared.cpp)
+add_exported_library(drmemtrace_opcode_mix STATIC tools/opcode_mix.cpp)
 add_exported_library(drmemtrace_syscall_mix STATIC tools/syscall_mix.cpp)
 add_exported_library(drmemtrace_view STATIC
                      tools/view.cpp tracer/raw2trace_shared.cpp)
 add_exported_library(drmemtrace_func_view STATIC tools/func_view.cpp)
 add_exported_library(drmemtrace_invariant_checker STATIC tools/invariant_checker.cpp)
 add_exported_library(drmemtrace_schedule_stats STATIC tools/schedule_stats.cpp)
+add_exported_library(drmemtrace_decode_cache STATIC
+                     tools/common/decode_cache.cpp
+                     tracer/raw2trace_shared.cpp)
 add_exported_library(drmemtrace_schedule_file STATIC common/schedule_file.cpp)
 add_exported_library(drmemtrace_mutex_dbg_owned STATIC common/mutex_dbg_owned.cpp)
 
-target_link_libraries(drmemtrace_invariant_checker drdecode drmemtrace_schedule_file)
+target_link_libraries(drmemtrace_invariant_checker drdecode drmemtrace_schedule_file
+                      drmemtrace_decode_cache)
+target_link_libraries(drmemtrace_decode_cache drcovlib_static)
+target_link_libraries(drmemtrace_opcode_mix drmemtrace_decode_cache)
 
+configure_DynamoRIO_standalone(drmemtrace_decode_cache)
 configure_DynamoRIO_standalone(drmemtrace_opcode_mix)
 configure_DynamoRIO_standalone(drmemtrace_view)
 configure_DynamoRIO_standalone(drmemtrace_invariant_checker)
@@ -320,6 +326,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/reader)
 # so that we can more cleanly separate tracer and raw2trace code.
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tracer)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/scheduler)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/tools/common)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 
 if (BUILD_PT_POST_PROCESSOR)
@@ -611,6 +618,7 @@ restore_nonclient_flags(drmemtrace_analyzer)
 restore_nonclient_flags(drmemtrace_invariant_checker)
 restore_nonclient_flags(drmemtrace_schedule_stats)
 restore_nonclient_flags(drmemtrace_schedule_file)
+restore_nonclient_flags(drmemtrace_decode_cache)
 
 # We need to pass /EHsc and we pull in libcmtd into drcachesim from a dep lib.
 # Thus we need to override the /MT with /MTd.
@@ -684,6 +692,7 @@ add_win32_flags(drmemtrace_analyzer)
 add_win32_flags(drmemtrace_invariant_checker)
 add_win32_flags(drmemtrace_schedule_stats)
 add_win32_flags(drmemtrace_schedule_file)
+add_win32_flags(drmemtrace_decode_cache)
 add_win32_flags(directory_iterator)
 add_win32_flags(test_helpers)
 add_win32_flags(drmemtrace_mutex_dbg_owned)
@@ -942,6 +951,26 @@ if (BUILD_TESTS)
     add_win32_flags(tool.drcacheoff.burst_aarch64_sys)
   endif ()
 
+  add_executable(tool.drcachesim.decode_cache_test tests/decode_cache_test.cpp)
+  configure_DynamoRIO_standalone(tool.drcachesim.decode_cache_test)
+  add_win32_flags(tool.drcachesim.decode_cache_test)
+  target_link_libraries(tool.drcachesim.decode_cache_test
+    drmemtrace_decode_cache test_helpers)
+  add_test(NAME tool.drcachesim.decode_cache_test
+           COMMAND tool.drcachesim.decode_cache_test)
+  set_tests_properties(tool.drcachesim.decode_cache_test PROPERTIES
+    TIMEOUT ${test_seconds})
+
+  add_executable(tool.drcacheoff.opcode_mix_test tests/opcode_mix_test.cpp)
+  configure_DynamoRIO_standalone(tool.drcacheoff.opcode_mix_test)
+  add_win32_flags(tool.drcacheoff.opcode_mix_test)
+  target_link_libraries(tool.drcacheoff.opcode_mix_test
+    drmemtrace_opcode_mix drmemtrace_decode_cache test_helpers)
+  add_test(NAME tool.drcacheoff.opcode_mix_test
+           COMMAND tool.drcacheoff.opcode_mix_test)
+  set_tests_properties(tool.drcacheoff.opcode_mix_test PROPERTIES
+    TIMEOUT ${test_seconds})
+
   # XXX i#1997: dynamorio_static is not supported on Mac yet
   # FIXME i#2949: gcc 7.3 fails to link certain configs
   # TODO i#3544: Port tests to RISC-V 64

clients/drcachesim/tests/decode_cache_test.cpp

@@ -0,0 +1,233 @@
+/* **********************************************************
+ * Copyright (c) 2024 Google, LLC  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of Google, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL GOOGLE, LLC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Tests for the decode_cache_t library. */
+
+#include <iostream>
+#include <vector>
+
+#include "decode_cache.h"
+#include "../common/memref.h"
+#include "memref_gen.h"
+
+namespace dynamorio {
+namespace drmemtrace {
+
+static constexpr addr_t TID_A = 1;
+static constexpr offline_file_type_t ENCODING_FILE_TYPE =
+    static_cast<offline_file_type_t>(OFFLINE_FILE_VERSION_ENCODINGS);
+
+class test_decode_info_t : public decode_info_base_t {
+public:
+    bool is_nop = false;
+    bool is_ret = false;
+
+private:
+    void
+    set_decode_info_derived(void *dcontext,
+                            const dynamorio::drmemtrace::_memref_instr_t &memref_instr,
+                            instr_t *instr) override
+    {
+        is_nop = instr_is_nop(instr);
+        is_ret = instr_is_return(instr);
+    }
+};
+
+std::string
+check_decode_caching(void *drcontext, bool persist_instrs, bool use_module_mapper)
+{
+    static constexpr addr_t BASE_ADDR = 0x123450;
+    instr_t *nop = XINST_CREATE_nop(drcontext);
+    instr_t *ret = XINST_CREATE_return(drcontext);
+    instrlist_t *ilist = instrlist_create(drcontext);
+    instrlist_append(ilist, nop);
+    instrlist_append(ilist, ret);
+    std::vector<memref_with_IR_t> memref_setup = {
+        { gen_instr(TID_A), nop },
+        { gen_instr(TID_A), ret },
+        { gen_instr(TID_A), nop },
+    };
+    std::vector<memref_t> memrefs;
+    instrlist_t *ilist_for_test_decode_cache = nullptr;
+    std::string module_file_for_test_decode_cache = "";
+    if (use_module_mapper) {
+        // This does not set encodings in the memref.instr.
+        memrefs = add_encodings_to_memrefs(ilist, memref_setup, 0,
+                                           /*set_only_instr_addr=*/true);
+        // We pass the instrs to construct the test_module_mapper_t in the
+        // test_decode_cache_t;
+        ilist_for_test_decode_cache = ilist;
+        module_file_for_test_decode_cache = "some_mod_file";
+    } else {
+        memrefs = add_encodings_to_memrefs(ilist, memref_setup, BASE_ADDR);
+    }
+    // Set up the second nop memref to reuse the same encoding as the first nop.
+    memrefs[2].instr.encoding_is_new = false;
+
+    test_decode_info_t test_decode_info;
+    if (test_decode_info.is_valid()) {
+        return "Unexpected valid default-constructed decode info";
+    }
+
+    if (persist_instrs) {
+        // These are tests to verify the operation of instr_decode_info_t: that it stores
+        // the instr_t correctly.
+        // Tests for instr_decode_cache_t are done when persist_instrs = false (see
+        // the else part below).
+        test_decode_cache_t<instr_decode_info_t> decode_cache(
+            drcontext,
+            /*persist_decoded_instr=*/true, ilist_for_test_decode_cache);
+        decode_cache.init(ENCODING_FILE_TYPE, module_file_for_test_decode_cache, "");
+        for (const memref_t &memref : memrefs) {
+            std::string err = decode_cache.add_decode_info(memref.instr);
+            if (err != "")
+                return err;
+        }
+        instr_decode_info_t *decode_info_nop =
+            decode_cache.get_decode_info(reinterpret_cast<app_pc>(memrefs[0].instr.addr));
+        if (decode_info_nop == nullptr || !decode_info_nop->is_valid() ||
+            !instr_is_nop(decode_info_nop->get_decoded_instr())) {
+            return "Unexpected instr_decode_info_t for nop instr";
+        }
+        instr_decode_info_t *decode_info_ret =
+            decode_cache.get_decode_info(reinterpret_cast<app_pc>(memrefs[1].instr.addr));
+        if (decode_info_ret == nullptr || !decode_info_ret->is_valid() ||
+            !instr_is_return(decode_info_ret->get_decoded_instr())) {
+            return "Unexpected instr_decode_info_t for ret instr";
+        }
+    } else {
+        // These are tests to verify the operation of instr_decode_cache_t: that it caches
+        // decode info correctly.
+        test_decode_cache_t<test_decode_info_t> decode_cache(
+            drcontext,
+            /*persist_decoded_instrs=*/false, ilist_for_test_decode_cache);
+        decode_cache.init(ENCODING_FILE_TYPE, module_file_for_test_decode_cache, "");
+        if (decode_cache.get_decode_info(
+                reinterpret_cast<app_pc>(memrefs[0].instr.addr)) != nullptr) {
+            return "Unexpected test_decode_info_t for never-seen pc";
+        }
+        std::string err = decode_cache.add_decode_info(memrefs[0].instr);
+        if (err != "")
+            return err;
+        test_decode_info_t *decode_info_nop =
+            decode_cache.get_decode_info(reinterpret_cast<app_pc>(memrefs[0].instr.addr));
+        if (decode_info_nop == nullptr || !decode_info_nop->is_valid() ||
+            !decode_info_nop->is_nop) {
+            return "Unexpected test_decode_info_t for nop instr";
+        }
+        err = decode_cache.add_decode_info(memrefs[1].instr);
+        if (err != "")
+            return err;
+        test_decode_info_t *decode_info_ret =
+            decode_cache.get_decode_info(reinterpret_cast<app_pc>(memrefs[1].instr.addr));
+        if (decode_info_ret == nullptr || !decode_info_ret->is_valid() ||
+            !decode_info_ret->is_ret) {
+            return "Unexpected test_decode_info_t for ret instr";
+        }
+        err = decode_cache.add_decode_info(memrefs[2].instr);
+        if (err != "")
+            return err;
+        test_decode_info_t *decode_info_nop_2 =
+            decode_cache.get_decode_info(reinterpret_cast<app_pc>(memrefs[2].instr.addr));
+        if (decode_info_nop_2 != decode_info_nop) {
+            return "Did not see same decode info instance for second instance of nop";
+        }
+    }
+    instrlist_clear_and_destroy(drcontext, ilist);
+    std::cerr << "check_decode_caching with persist_instrs: " << persist_instrs
+              << ", use_module_mapper: " << use_module_mapper << " passed\n";
+    return "";
+}
+
+std::string
+check_missing_module_mapper_and_no_encoding(void *drcontext)
+{
+    memref_t instr = gen_instr(TID_A);
+    test_decode_cache_t<instr_decode_info_t> decode_cache(
+        drcontext,
+        /*persist_decoded_instr=*/true, /*ilist_for_test_module_mapper=*/nullptr);
+    std::string err = decode_cache.add_decode_info(instr.instr);
+    if (err == "") {
+        return "Expected error at add_decode_info but did not get any";
+    }
+    err = decode_cache.init(
+        static_cast<offline_file_type_t>(OFFLINE_FILE_TYPE_SYSCALL_NUMBERS), "", "");
+    if (err == "") {
+        return "Expected error at init but did not get any";
+    }
+    std::cerr << "check_missing_module_mapper passed\n";
+    return "";
+}
+
+int
+test_main(int argc, const char *argv[])
+{
+    void *drcontext = dr_standalone_init();
+    std::string err = check_decode_caching(drcontext, /*persist_instrs=*/false,
+                                           /*use_module_mapper=*/false);
+    if (err != "") {
+        std::cerr << err << "\n";
+        exit(1);
+    }
+    err = check_decode_caching(drcontext, /*persist_instrs=*/true,
+                               /*use_module_mapper=*/false);
+    if (err != "") {
+        std::cerr << err << "\n";
+        exit(1);
+    }
+#ifndef WINDOWS
+    // TODO i#5960: Enable these tests after the test-only Windows issue is
+    // fixed.
+    err = check_decode_caching(drcontext, /*persist_instrs=*/false,
+                               /*use_module_mapper=*/true);
+    if (err != "") {
+        std::cerr << err << "\n";
+        exit(1);
+    }
+    err = check_decode_caching(drcontext, /*persist_instrs=*/true,
+                               /*use_module_mapper=*/true);
+    if (err != "") {
+        std::cerr << err << "\n";
+        exit(1);
+    }
+#endif
+    err = check_missing_module_mapper_and_no_encoding(drcontext);
+    if (err != "") {
+        std::cerr << err << "\n";
+        exit(1);
+    }
+
+    return 0;
+}
+
+} // namespace drmemtrace
+} // namespace dynamorio

clients/drcachesim/tests/memref_gen.h

@@ -188,7 +188,7 @@ gen_exit(memref_tid_t tid)
 std::vector<memref_t>
 add_encodings_to_memrefs(instrlist_t *ilist,
                          std::vector<memref_with_IR_t> &memref_instr_vec,
-                         addr_t base_addr)
+                         addr_t base_addr, bool set_only_instr_addr = false)
 {
     static const int MAX_DECODE_SIZE = 2048;
     byte decode_buf[MAX_DECODE_SIZE];
@@ -207,8 +207,10 @@ add_encodings_to_memrefs(instrlist_t *ilist,
             const int instr_size = instr_length(GLOBAL_DCONTEXT, pair.instr);
             pair.memref.instr.addr = offset + base_addr;
             pair.memref.instr.size = instr_size;
-            memcpy(pair.memref.instr.encoding, &decode_buf[offset], instr_size);
-            pair.memref.instr.encoding_is_new = true;
+            if (!set_only_instr_addr) {
+                memcpy(pair.memref.instr.encoding, &decode_buf[offset], instr_size);
+                pair.memref.instr.encoding_is_new = true;
+            }
         } else if (pair.memref.marker.type == TRACE_TYPE_MARKER &&
                    pair.instr != nullptr) {
             pair.memref.marker.marker_value = instr_get_offset(pair.instr) + base_addr;