i#731,i#3271: Keep IR cti+copied bits valid and re-relativized (#4018)

Adds rip-relative information tracking at IR levels 1-3 for
instruction references on x86, extending the existing tracking for
data references.

Adds decode_sizeof_ex() and instr_get_rel_data_or_instr_target() to
support rip-relative displacement and target handling.

Changes decode_from_copy() to preserve raw bits for all instructions,
eliminating problems where untracked encoding features are lost such
as in #4017.

Documents the changes and describes how to avoid the new behavior.
Does just that for intra-sequence cti's in the rseq native code copy and
Windows syscall wrapper copies.

Adds test cases to api.ir.

Implementing the same thing for AArchXX is left unimplemented, tracked
by i#4016.

Issue: #731, #3271, #3339, #4016, #4017
Fixes #731
Fixes #3271
Fixes #4017

api/docs/intro.dox

@@ -1,5 +1,5 @@
 /* ******************************************************************************
- * Copyright (c) 2010-2019 Google, Inc.  All rights reserved.
+ * Copyright (c) 2010-2020 Google, Inc.  All rights reserved.
  * Copyright (c) 2011 Massachusetts Institute of Technology  All rights reserved.
  * Copyright (c) 2007-2010 VMware, Inc.  All rights reserved.
  * ******************************************************************************/
@@ -2078,19 +2078,6 @@ When using DynamoRIO's CMake support, use the configure_DynamoRIO_decoder()
 function to set up include directories and to link with \p drdecode.  The
 next section describes how to link with the DynamoRIO shared library.
 
-\subsection sec_relativize Re-Relativization of Jumps and Calls
-
-When encoding a relative jump or call to a different location than it was
-decoded from while in standalone mode, a re-encode must be forced in order
-to work around an issue where DynamoRIO does not re-relativize the target:
-
-\code
-instr_set_raw_bits_valid(instr, false)
-\endcode
-
-When not in standalone mode, all branches are mangled and thus this is
-never an issue.  This should be fixed in a future release.
-
 
 \section sec_standalone_shared DynamoRIO Shared Library Issues
 

api/docs/release.dox

@@ -175,6 +175,12 @@ compatibility changes:
  - Changed the #PFX format string specifier to use %p rather than %x internally.
  - DR no longer forwards _snprintf, _snwprintf, _vsnprintf, sprintf, or sscanf to
    ntdll.  Clients should use the dr_-prefixed versions of these functions.
+ - PC-relative control transfer instructions are now auto-re-relativized by the
+   general decoder and encoder.  This affects clients and standalone tools that use
+   decode_from_copy() or instr_encode_to_copy() or instrlist_encode_to_copy().
+   Previously, re-relativization for instruction references only happened when an
+   instruction was re-encoded.  This auto-PC-relativization can be avoided by calling
+   instr_set_rip_rel_valid() and setting the validity of the PC-relative data to false.
 
 Further non-compatibility-affecting changes include:
 
@@ -225,6 +231,8 @@ Further non-compatibility-affecting changes include:
  - Added drmgr_is_first_nonlabel_instr() and instrlist_first_nonlabel().
  - Added a new standalone tool "drdisas" which disassembles raw bytes using
    DR's decoder.
+ - Added decode_sizeof_ex() and instr_get_rel_data_or_instr_target() handling
+   relative instruction references.
 
 **************************************************
 <hr>

core/arch/aarch64/codec.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2017 Google, Inc.  All rights reserved.
+ * Copyright (c) 2017-2020 Google, Inc.  All rights reserved.
  * Copyright (c) 2016 ARM Limited. All rights reserved.
  * **********************************************************/
 
@@ -3161,6 +3161,7 @@ decode_common(dcontext_t *dcontext, byte *pc, byte *orig_pc, instr_t *instr)
     if (orig_pc != pc) {
         /* We do not want to copy when encoding and condone an invalid
          * relative target.
+         * TODO i#4016: Add re-relativization support without having to re-encode.
          */
         instr_set_raw_bits_valid(instr, false);
         instr_set_translation(instr, orig_pc);

core/arch/aarch64/encode.c

@@ -1,4 +1,5 @@
 /* **********************************************************
+ * Copyright (c) 2020 Google, Inc. All rights reserved.
  * Copyright (c) 2016 ARM Limited. All rights reserved.
  * **********************************************************/
 
@@ -193,7 +194,7 @@ byte *
 copy_and_re_relativize_raw_instr(dcontext_t *dcontext, instr_t *instr, byte *dst_pc,
                                  byte *final_pc)
 {
-    /* FIXME i#1569: re-relativizing is NYI */
+    /* TODO i#4016: re-relativizing is NYI */
     /* OP_ldstex is always relocatable. */
     ASSERT(instr_raw_bits_valid(instr) || instr_get_opcode(instr) == OP_ldstex);
     memcpy(dst_pc, instr->bytes, instr->length);

core/arch/arm/decode.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2014-2019 Google, Inc.  All rights reserved.
+ * Copyright (c) 2014-2020 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -2500,6 +2500,7 @@ decode_common(dcontext_t *dcontext, byte *pc, byte *orig_pc, instr_t *instr)
     if (orig_pc != pc) {
         /* We do not want to copy when encoding and condone an invalid
          * relative target
+         * TODO i#4016: Add re-relativization support without having to re-encode.
          */
         instr_set_raw_bits_valid(instr, false);
         instr_set_translation(instr, orig_pc);

core/arch/arm/encode.c

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2014-2015 Google, Inc.  All rights reserved.
+ * Copyright (c) 2014-2020 Google, Inc.  All rights reserved.
  * **********************************************************/
 
 /*
@@ -3099,7 +3099,7 @@ byte *
 copy_and_re_relativize_raw_instr(dcontext_t *dcontext, instr_t *instr, byte *dst_pc,
                                  byte *final_pc)
 {
-    /* FIXME i#1551: re-relativizing is NYI */
+    /* TODO i#4016: re-relativizing is NYI */
     ASSERT(instr_raw_bits_valid(instr));
     memcpy(dst_pc, instr->bytes, instr->length);
     return dst_pc + instr->length;

core/arch/decode_fast.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2015-2019 Google, Inc.  All rights reserved.
+ * Copyright (c) 2015-2020 Google, Inc.  All rights reserved.
  * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -46,15 +46,37 @@ DR_API
  * Decodes only enough of the instruction at address \p pc to determine its size.
  * Returns that size.
  * If \p num_prefixes is non-NULL, returns the number of prefix bytes.
- * If \p rip_rel_pos is non-NULL, returns the offset into the instruction
- * of a rip-relative addressing displacement (for data only: ignores
- * control-transfer relative addressing), or 0 if none.
+ *
+ * On x86, if \p rip_rel_pos is non-NULL, returns the offset into the instruction of a
+ * rip-relative addressing displacement (for data only: ignores control-transfer
+ * relative addressing; use decode_sizeof_ex() for that), or 0 if none.
+ * The \p rip_rel_pos parameter is only implemented for x86, where the displacement
+ * is always 4 bytes in size.
+ *
  * May return 0 size for certain invalid instructions.
  */
 int
 decode_sizeof(dcontext_t *dcontext, byte *pc,
               int *num_prefixes _IF_X86_64(uint *rip_rel_pos));
 
+#ifdef X86
+DR_API
+/**
+ * Decodes only enough of the instruction at address \p pc to determine its size.
+ * Returns that size.
+ * If \p num_prefixes is non-NULL, returns the number of prefix bytes.
+ *
+ * On x86, if \p rip_rel_pos is non-NULL, returns the offset into the instruction of a
+ * rip-relative addressing displacement for data or control-transfer relative
+ * addressing, or 0 if none.  This is only implemented for x86, where the displacement
+ * is always 4 bytes for data but can be 1 byte for control.
+ *
+ * May return 0 size for certain invalid instructions.
+ */
+int
+decode_sizeof_ex(dcontext_t *dcontext, byte *pc, int *num_prefixes, uint *rip_rel_pos);
+#endif
+
 DR_API
 /**
  * Decodes only enough of the instruction at address \p pc to determine its size.
@@ -99,7 +121,7 @@ DR_UNS_EXCEPT_TESTS_API
  * Does NOT fill in any other prefix flags unless this is a cti instr
  * and the flags affect the instr.
  *
- * For x64, calls instr_set_rip_rel_pos().  Thus, if the raw bytes are
+ * For x86, calls instr_set_rip_rel_pos().  Thus, if the raw bytes are
  * not modified prior to encode time, any rip-relative offset will be
  * automatically re-relativized (though encoding will fail if the new
  * encode location cannot reach the original target).

core/arch/instr.h

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2011-2019 Google, Inc.  All rights reserved.
+ * Copyright (c) 2011-2020 Google, Inc.  All rights reserved.
  * Copyright (c) 2000-2010 VMware, Inc.  All rights reserved.
  * **********************************************************/
 
@@ -460,7 +460,7 @@ struct _instr_t {
 
     uint opcode;
 
-#    ifdef X86_64
+#    ifdef X86
     /* PR 251479: offset into instr's raw bytes of rip-relative 4-byte displacement */
     byte rip_rel_pos;
 #    endif
@@ -1870,12 +1870,24 @@ bool
 instr_is_xsave(instr_t *instr);
 #endif
 
+DR_API
+/**
+ * If any of \p instr's operands is a rip-relative data or instruction
+ * memory reference, returns the address that reference targets.  Else
+ * returns false.  For instruction references, only PC operands are
+ * considered: not instruction pointer operands.
+ *
+ * \note Currently this is only implemented for x86.
+ */
+bool
+instr_get_rel_data_or_instr_target(instr_t *instr, /*OUT*/ app_pc *target);
+
 /* DR_API EXPORT BEGIN */
 #if defined(X64) || defined(ARM)
 /* DR_API EXPORT END */
 DR_API
 /**
- * Returns true iff any of \p instr's operands is a rip-relative memory reference.
+ * Returns true iff any of \p instr's operands is a rip-relative data memory reference.
  *
  * \note For 64-bit DR builds only.
  */
@@ -1884,7 +1896,7 @@ instr_has_rel_addr_reference(instr_t *instr);
 
 DR_API
 /**
- * If any of \p instr's operands is a rip-relative memory reference, returns the
+ * If any of \p instr's operands is a rip-relative data memory reference, returns the
  * address that reference targets.  Else returns false.
  *
  * \note For 64-bit DR builds only.
@@ -1894,7 +1906,7 @@ instr_get_rel_addr_target(instr_t *instr, /*OUT*/ app_pc *target);
 
 DR_API
 /**
- * If any of \p instr's destination operands is a rip-relative memory
+ * If any of \p instr's destination operands is a rip-relative data memory
  * reference, returns the operand position.  If there is no such
  * destination operand, returns -1.
  *
@@ -1917,7 +1929,7 @@ instr_get_rel_addr_src_idx(instr_t *instr);
 #endif /* X64 || ARM */
 /* DR_API EXPORT END */
 
-#ifdef X86_64
+#ifdef X86
 /* We're not exposing the low-level rip_rel_pos routines directly to clients,
  * who should only use this level 1-3 feature via decode_cti + encode.
  */
@@ -1949,7 +1961,7 @@ instr_get_rip_rel_pos(instr_t *instr);
  */
 void
 instr_set_rip_rel_pos(instr_t *instr, uint pos);
-#endif /* X64 */
+#endif /* X86 */
 
 /* not exported: for PR 267260 */
 bool
@@ -3007,7 +3019,7 @@ instr_create_save_to_reg(dcontext_t *dcontext, reg_id_t reg1, reg_id_t reg2);
 instr_t *
 instr_create_restore_from_reg(dcontext_t *dcontext, reg_id_t reg1, reg_id_t reg2);
 
-#ifdef X64
+#ifdef X86_64
 byte *
 instr_raw_is_rip_rel_lea(byte *pc, byte *read_end);
 #endif