Skip to content

Commit

Permalink
i#5659: Handle multiple app copies in rseq mangling
Browse files Browse the repository at this point in the history 
Adds support for multiple app copies in rseq mangling finalization's
search for labels to fill in code cache PC's.

Adds support for multiple rseq regions in one fragment_t by expanding
the stored rseq_cs per fragment to become a list.

Adds "-trace_after_instrs 5K" to the tool.drcacheoff.rseq test to
exercise rseq with drbbdup.

Augments the attach/detach api.rseq test to use drmemtrace, which with
the forthcoming #2039 mode switches also hits this multi-rseq case.

Fixes #5659
  • Loading branch information
@derekbruening
derekbruening committed Sep 26, 2022
1 parent 44312ad commit 1757ea9
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 58 deletions.
50 changes: 33 additions & 17 deletions core/arch/mangle_shared.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1404,14 +1404,13 @@ mangle_rseq_finalize(dcontext_t *dcontext, instrlist_t *ilist, fragment_t *f)
instr_t *instr, *immed_first = NULL, *immed_last = NULL;
cache_pc pc = FCACHE_ENTRY_PC(f), immed_start_pc = NULL;
cache_pc rseq_start = NULL, rseq_end = NULL, rseq_abort = NULL;
DEBUG_DECLARE(int label_sets_found = 0;)
for (instr = instrlist_first(ilist); instr != NULL; instr = instr_get_next(instr)) {
if (instr_is_label(instr) &&
(instr_get_note(instr) == (void *)DR_NOTE_RSEQ ||
TEST(INSTR_RSEQ_ENDPOINT, instr->flags))) {
dr_instr_label_data_t *label_data = instr_get_label_data_area(instr);
switch (label_data->data[0]) {
case DR_RSEQ_LABEL_START: rseq_start = pc; break;
case DR_RSEQ_LABEL_END: rseq_end = pc; break;
case DR_RSEQ_LABEL_ABORT: rseq_abort = pc; break;
case DR_RSEQ_LABEL_CS:
immed_start_pc = pc;
Expand All @@ -1428,26 +1427,43 @@ mangle_rseq_finalize(dcontext_t *dcontext, instrlist_t *ilist, fragment_t *f)
}
}
break;
case DR_RSEQ_LABEL_START: rseq_start = pc; break;
case DR_RSEQ_LABEL_END: {
rseq_end = pc;
/* We assume this is the 4th and last label. We handle it here,
* so we can start over on a new set if there are multiple rseq
* regions (such as from duplicated app copies by drbbdup).
*/
IF_DEBUG(++label_sets_found;)
ASSERT(rseq_start != NULL && rseq_abort != NULL);
byte *rseq_cs_alloc, *rseq_cs;
/* The rseq_cs creation and recording is structured like this in two steps
* to provide flexibility in mangling. Originally the alloc was done in
* mangle_rseq() and passed here in the label data, but to simplify
* freeing we now allocate here and patch the immediates.
*/
rseq_cs_alloc = rseq_get_rseq_cs_alloc(&rseq_cs);
rseq_record_rseq_cs(rseq_cs_alloc, f, rseq_start, rseq_end, rseq_abort);
ASSERT(immed_start_pc != NULL && immed_first != NULL);
LOG(THREAD, LOG_INTERP, 4, "%s: start=%p, end=%p, abort=%p stored @%p\n",
__FUNCTION__, rseq_start, rseq_end, rseq_abort, rseq_cs);
patch_mov_immed_ptrsz(dcontext, (ptr_int_t)rseq_cs, immed_start_pc,
immed_first, immed_last);
DODEBUG({
rseq_abort = NULL;
rseq_start = NULL;
immed_start_pc = NULL;
immed_first = NULL;
});
break;
}
default: ASSERT_NOT_REACHED();
}
}
pc += instr_length(dcontext, instr);
}
ASSERT(rseq_start != NULL && rseq_end != NULL && rseq_abort != NULL);

byte *rseq_cs_alloc, *rseq_cs;
/* The rseq_cs creation and recording is structured like this in two steps to
* provide flexibility in mangling. Originally the alloc was done in mangle_rseq()
* and passed here in the label data, but to simplify freeing we now allocate here
* and patch the immediates.
*/
rseq_cs_alloc = rseq_get_rseq_cs_alloc(&rseq_cs);
rseq_record_rseq_cs(rseq_cs_alloc, f, rseq_start, rseq_end, rseq_abort);
ASSERT(immed_start_pc != NULL && immed_first != NULL);
LOG(THREAD, LOG_INTERP, 4, "%s: start=%p, end=%p, abort=%p stored @%p\n",
__FUNCTION__, rseq_start, rseq_end, rseq_abort, rseq_cs);
patch_mov_immed_ptrsz(dcontext, (ptr_int_t)rseq_cs, immed_start_pc, immed_first,
immed_last);
/* We should have found at least one set of labels. */
ASSERT(label_sets_found > 0);
}
#endif /* LINUX */

Expand Down
35 changes: 29 additions & 6 deletions core/unix/rseq_linux.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,14 @@ typedef struct _rseq_region_t {
bool reg_written[DR_NUM_GPR_REGS];
} rseq_region_t;

/* We need to store a struct rseq_cs per fragment_t. To avoid the cost of adding a
/* We need to store potentially multiple rseq_cs per fragment. */
typedef struct _rseq_cs_record_t {
struct rseq_cs rcs;
void *alloc_ptr;
struct _rseq_cs_record_t *next;
} rseq_cs_record_t;

/* We need to store an rseq_cs_record_t per fragment_t. To avoid the cost of adding a
* pointer field to every fragment_t, and the complexity of another subclass like
* trace_t, we store them externally in a hashtable. The FRAG_HAS_RSEQ_ENDPOINT flag
* avoids the hashtable lookup on every fragment.
Expand Down Expand Up @@ -128,13 +135,18 @@ rseq_area_dup(void *data)
static inline size_t
rseq_cs_alloc_size(void)
{
return sizeof(struct rseq) + __alignof(struct rseq_cs);
return sizeof(rseq_cs_record_t) + __alignof(struct rseq_cs);
}

static void
rseq_cs_free(dcontext_t *dcontext, void *data)
{
global_heap_free(data, rseq_cs_alloc_size() HEAPACCT(ACCT_OTHER));
rseq_cs_record_t *record = (rseq_cs_record_t *)data;
do {
void *tofree = record->alloc_ptr;
record = record->next;
global_heap_free(tofree, rseq_cs_alloc_size() HEAPACCT(ACCT_OTHER));
} while (record != NULL);
}

void
Expand Down Expand Up @@ -244,15 +256,26 @@ void
rseq_record_rseq_cs(byte *rseq_cs_alloc, fragment_t *f, cache_pc start, cache_pc end,
cache_pc abort)
{
struct rseq_cs *target =
(struct rseq_cs *)ALIGN_FORWARD(rseq_cs_alloc, __alignof(struct rseq_cs));
rseq_cs_record_t *record =
(rseq_cs_record_t *)ALIGN_FORWARD(rseq_cs_alloc, __alignof(struct rseq_cs));
record->alloc_ptr = rseq_cs_alloc;
record->next = NULL;
struct rseq_cs *target = &record->rcs;
target->version = 0;
target->flags = 0;
target->start_ip = (ptr_uint_t)start;
target->post_commit_offset = (ptr_uint_t)(end - start);
target->abort_ip = (ptr_uint_t)abort;
TABLE_RWLOCK(rseq_cs_table, write, lock);
generic_hash_add(GLOBAL_DCONTEXT, rseq_cs_table, (ptr_uint_t)f, rseq_cs_alloc);
rseq_cs_record_t *existing =
generic_hash_lookup(GLOBAL_DCONTEXT, rseq_cs_table, (ptr_uint_t)f);
if (existing != NULL) {
while (existing->next != NULL)
existing = existing->next;
existing->next = record;
} else {
generic_hash_add(GLOBAL_DCONTEXT, rseq_cs_table, (ptr_uint_t)f, record);
}
TABLE_RWLOCK(rseq_cs_table, write, unlock);
}

Expand Down
7 changes: 6 additions & 1 deletion suite/tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3818,7 +3818,10 @@ if (BUILD_CLIENTS)
endif ()

if (LINUX AND X64 AND HAVE_RSEQ)
torunonly_drcacheoff(rseq linux.rseq ""
torunonly_drcacheoff(rseq linux.rseq
# Run with -trace_after_instrs to ensure we test the
# drbbdup + rseq combo (i#5658, i#5659).
"-trace_after_instrs 5K"
"@${test_mode_flag}@-test_mode_name@rseq_app" "")
endif ()

Expand Down Expand Up @@ -4673,6 +4676,8 @@ if (UNIX)
tobuild_api(api.rseq linux/rseq.c "" "" OFF ON OFF)
link_with_pthread(api.rseq)
append_property_list(TARGET api.rseq COMPILE_DEFINITIONS "RSEQ_TEST_ATTACH")
use_DynamoRIO_static_client(api.rseq drmemtrace_static)
use_DynamoRIO_drmemtrace_tracer(api.rseq)
set(api.rseq_expectbase rseq_client)
# Test non-compliant code with our workaround flag.
tobuild_ops(linux.rseq_disable linux/rseq_disable.c "-disable_rseq" "")
Expand Down
68 changes: 34 additions & 34 deletions suite/tests/linux/rseq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
# include "thread.h"
# include "condvar.h"
# include <stdatomic.h>
# include "drmemtrace/drmemtrace.h"
#endif
#ifndef LINUX
# error Only Linux is supported.
Expand Down Expand Up @@ -180,12 +181,11 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o
"5:\n\t"
"movq $0, %[rseq_tls]\n\t"
/* clang-format on */
: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ id ] "=m"(id),
[ completions ] "=m"(completions), [ restarts ] "=m"(restarts),
[ force_restart_write ] "=m"(force_restart)
: [ cpu_id ] "m"(rseq_tls.cpu_id),
[ cpu_id_uninit ] "i"(RSEQ_CPU_ID_UNINITIALIZED),
[ force_restart ] "m"(force_restart)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id),
[completions] "=m"(completions), [restarts] "=m"(restarts),
[force_restart_write] "=m"(force_restart)
: [cpu_id] "m"(rseq_tls.cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED),
[force_restart] "m"(force_restart)
: "rax", "memory");
#elif defined(AARCH64)
__asm__ __volatile__(
Expand Down Expand Up @@ -246,12 +246,11 @@ test_rseq_call_once(bool force_restart_in, int *completions_out, int *restarts_o
"5:\n\t"
"str xzr, %[rseq_tls]\n\t"
/* clang-format on */
: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ id ] "=m"(id),
[ completions ] "=m"(completions), [ restarts ] "=m"(restarts),
[ force_restart_write ] "=m"(force_restart)
: [ cpu_id ] "m"(rseq_tls.cpu_id),
[ cpu_id_uninit ] "i"(RSEQ_CPU_ID_UNINITIALIZED),
[ force_restart ] "m"(force_restart)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id),
[completions] "=m"(completions), [restarts] "=m"(restarts),
[force_restart_write] "=m"(force_restart)
: [cpu_id] "m"(rseq_tls.cpu_id), [cpu_id_uninit] "i"(RSEQ_CPU_ID_UNINITIALIZED),
[force_restart] "m"(force_restart)
: "x0", "x1", "memory");
#else
# error Unsupported arch
Expand Down Expand Up @@ -340,10 +339,10 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_
"movq $0, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ id ] "=m"(id),
[ completions ] "=m"(completions), [ restarts ] "=m"(restarts),
[ force_restart_write ] "=m"(force_restart)
: [ cpu_id ] "m"(rseq_tls.cpu_id), [ force_restart ] "m"(force_restart)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id),
[completions] "=m"(completions), [restarts] "=m"(restarts),
[force_restart_write] "=m"(force_restart)
: [cpu_id] "m"(rseq_tls.cpu_id), [force_restart] "m"(force_restart)
: "rax", "rcx", "memory");
#elif defined(AARCH64)
__asm__ __volatile__(
Expand Down Expand Up @@ -410,10 +409,10 @@ test_rseq_branches_once(bool force_restart, int *completions_out, int *restarts_
"str xzr, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ id ] "=m"(id),
[ completions ] "=m"(completions), [ restarts ] "=m"(restarts),
[ force_restart_write ] "=m"(force_restart)
: [ cpu_id ] "m"(rseq_tls.cpu_id), [ force_restart ] "m"(force_restart)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id),
[completions] "=m"(completions), [restarts] "=m"(restarts),
[force_restart_write] "=m"(force_restart)
: [cpu_id] "m"(rseq_tls.cpu_id), [force_restart] "m"(force_restart)
: "x0", "x1", "memory");
#else
# error Unsupported arch
Expand Down Expand Up @@ -493,7 +492,7 @@ test_rseq_native_fault(void)
"movq $0, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ restarts ] "=m"(restarts)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts)
:
: "rax", "rcx", "xmm0", "xmm1", "memory");
#elif defined(AARCH64)
Expand Down Expand Up @@ -547,7 +546,7 @@ test_rseq_native_fault(void)
"str xzr, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ restarts ] "=m"(restarts)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts)
:
: "x0", "x1", "q0", "q1", "memory");
#else
Expand Down Expand Up @@ -633,9 +632,9 @@ test_rseq_native_abort(void)
"movq $0, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ restarts ] "=m"(restarts)
: [ cpu_mask_size ] "i"(sizeof(cpu_set_t)),
[ sysnum_setaffinity ] "i"(SYS_sched_setaffinity)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts)
: [cpu_mask_size] "i"(sizeof(cpu_set_t)),
[sysnum_setaffinity] "i"(SYS_sched_setaffinity)
: "rax", "rcx", "rdx", "xmm0", "xmm1", "memory");
# elif defined(AARCH64)
__asm__ __volatile__(
Expand Down Expand Up @@ -706,9 +705,9 @@ test_rseq_native_abort(void)
"str xzr, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ restarts ] "=m"(restarts)
: [ cpu_mask_size ] "i"(sizeof(cpu_set_t)),
[ sysnum_setaffinity ] "i"(SYS_sched_setaffinity)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [restarts] "=m"(restarts)
: [cpu_mask_size] "i"(sizeof(cpu_set_t)),
[sysnum_setaffinity] "i"(SYS_sched_setaffinity)
: "x0", "x1", "x2", "x8", "q0", "q1", "memory");
# else
# error Unsupported arch
Expand Down Expand Up @@ -770,8 +769,8 @@ test_rseq_writeback_store(void)
"str xzr, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ id ] "=m"(id), [ index ] "=g"(index)
: [ cpu_id ] "m"(rseq_tls.cpu_id)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [id] "=m"(id), [index] "=g"(index)
: [cpu_id] "m"(rseq_tls.cpu_id)
: "x0", "x1", "x2", "x28", "memory");
assert(id != RSEQ_CPU_ID_UNINITIALIZED);
}
Expand Down Expand Up @@ -837,8 +836,8 @@ rseq_thread_loop(void *arg)
"movq $0, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ zero ] "=m"(zero)
: [ exit_requested ] "m"(exit_requested)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [zero] "=m"(zero)
: [exit_requested] "m"(exit_requested)
: "rax", "memory");
# elif defined(AARCH64)
__asm__ __volatile__(
Expand Down Expand Up @@ -888,8 +887,8 @@ rseq_thread_loop(void *arg)
"str xzr, %[rseq_tls]\n\t"
/* clang-format on */

: [ rseq_tls ] "=m"(rseq_tls.rseq_cs), [ zero ] "=m"(zero)
: [ exit_requested ] "m"(exit_requested)
: [rseq_tls] "=m"(rseq_tls.rseq_cs), [zero] "=m"(zero)
: [exit_requested] "m"(exit_requested)
: "x0", "x1", "memory");
# else
# error Unsupported arch
Expand Down Expand Up @@ -948,6 +947,7 @@ dr_client_main(client_id_t id, int argc, const char *argv[])
{
/* Ensure DR_XFER_RSEQ_ABORT is rasied. */
dr_register_kernel_xfer_event(kernel_xfer_event);
drmemtrace_client_main(id, argc, argv);
}
#endif /* RSEQ_TEST_ATTACH */

Expand Down

0 comments on commit 1757ea9

Please sign in to comment.