Fix out-of-bounds access in AVX register handling on some CPUs

[lwintermelon]

For cpu like intel N5095, supported_feature_bits is 0b11, it doesn't support any AVX.

rr/src/ExtraRegisters.cc

Lines 139 to 154 in 18077ca

	static bool reg_is_avx2_or_512(GdbServerRegister reg, RegData& out) noexcept {
	if (reg < DREG_64_YMM0H || reg > DREG_64_K7) {
	return false;
	}
	for (const auto& descriptor : register_config_lookup_table) {
	if (descriptor.describes_register(reg)) {
	out.xsave_feature_bit = descriptor.feature;
	out.size = descriptor.size;
	out.offset = descriptor.register_offset(reg);
	return true;
	}
	}
	FATAL() << "Unknown AVX512F register: " << reg;
	return true;
	}

rr/src/ExtraRegisters.cc

Lines 96 to 101 in 18077ca

	int register_offset(GdbServerRegister reg) const noexcept {
	DEBUG_ASSERT(reg >= base && reg < (base + 16));
	const auto& layout = xsave_native_layout();
	return layout.feature_layouts[feature].offset + hi16_offset +
	(reg - base) * stride;
	}

So layout.feature_layouts[feature] above will make out-of-bounds access if feature is 2.

Any better solutions to fix this?

[khuey]

Why is gdb asking for AVX registers if the machine doesn't support them?

[lwintermelon]

The backtrace

Program received signal SIGSEGV, Segmentation fault.
rr::RegisterDescriptor::register_offset (this=0x555555c47660 <rr::register_config_lookup_table>, reg=rr::DREG_K2) at /home/test/project/rr/src/ExtraRegisters.cc:99
99          return layout.feature_layouts[feature].offset + hi16_offset +
(gdb) bt
#0  rr::RegisterDescriptor::register_offset (this=0x555555c47660 <rr::register_config_lookup_table>, reg=rr::DREG_K2) at /home/test/project/rr/src/ExtraRegisters.cc:99
#1  0x00005555559327bf in rr::reg_is_avx2_or_512 (reg=rr::DREG_K2, out=...) at /home/test/project/rr/src/ExtraRegisters.cc:148
#2  0x0000555555932a8a in rr::xsave_register_data (arch=rr::x86_64, regno=rr::DREG_K2) at /home/test/project/rr/src/ExtraRegisters.cc:209
#3  0x0000555555932f14 in rr::ExtraRegisters::read_register (this=0x555555e25500, buf=0x7fffffffd550 "\200\325\377\377\377\177", regno=rr::DREG_K2, 
    defined=0x7fffffffd406) at /home/test/project/rr/src/ExtraRegisters.cc:294
#4  0x0000555555933c36 in rr::get_full_value (r=..., low=rr::DREG_MXCSR, hi=rr::DREG_K2, buf=0x7fffffffd540 ".\v*\t\200\200\200\200")
    at /home/test/project/rr/src/ExtraRegisters.cc:469
#5  0x00005555559366a7 in rr::compare_regs (reg1=..., reg2=..., low=rr::DREG_MXCSR, hi=rr::DREG_K2, num_regs=8, name_base=0x555555c4787e "ymm", result=...)
    at /home/test/project/rr/src/ExtraRegisters.cc:893
#6  0x0000555555936b09 in rr::ExtraRegisters::compare_internal (this=0x555555e25500, reg2=..., result=...) at /home/test/project/rr/src/ExtraRegisters.cc:933
#7  0x0000555555af4d2b in rr::ExtraRegisters::matches (this=0x555555e25500, reg2=...) at /home/test/project/rr/src/ExtraRegisters.h:167
#8  0x0000555555aecbbe in rr::is_same_execution_point (t=0x555555e25200, rec_regs=..., rec_extra_regs=..., ticks_left=0, mismatched_regs=0x7fffffffd890, 
    mismatched_regs_ptr=0x7fffffffd840, in_syscallbuf=false) at /home/test/project/rr/src/ReplaySession.cc:941
#9  0x0000555555aed2d1 in rr::ReplaySession::emulate_async_signal (this=0x555555e1d9a0, t=0x555555e25200, constraints=..., ticks=3375360, in_syscallbuf_syscall_hook=...)
    at /home/test/project/rr/src/ReplaySession.cc:1056
#10 0x0000555555af1288 in rr::ReplaySession::try_one_trace_step (this=0x555555e1d9a0, t=0x555555e25200, constraints=...)
    at /home/test/project/rr/src/ReplaySession.cc:1732
#11 0x0000555555af2bdf in rr::ReplaySession::replay_step (this=0x555555e1d9a0, constraints=...) at /home/test/project/rr/src/ReplaySession.cc:2058
#12 0x0000555555ae6bb3 in rr::ReplaySession::replay_step (this=0x555555e1d9a0, command=rr::RUN_CONTINUE) at /home/test/project/rr/src/ReplaySession.h:294
#13 0x0000555555ae4ed7 in rr::serve_replay_no_debugger (trace_dir=..., flags=...) at /home/test/project/rr/src/ReplayCommand.cc:402
#14 0x0000555555ae5931 in rr::replay (trace_dir=..., flags=...) at /home/test/project/rr/src/ReplayCommand.cc:512
#15 0x0000555555ae687c in rr::ReplayCommand::run (this=0x555555e09850 <rr::ReplayCommand::singleton>, args=...) at /home/test/project/rr/src/ReplayCommand.cc:695
#16 0x00005555559dc1d2 in main (argc=5, argv=0x7fffffffe5d8) at /home/test/project/rr/src/main.cc:278
(gdb) list
94        int stride;
95
96        int register_offset(GdbServerRegister reg) const noexcept {
97          DEBUG_ASSERT(reg >= base && reg < (base + 16));
98          const auto& layout = xsave_native_layout();
99          return layout.feature_layouts[feature].offset + hi16_offset +
100                (reg - base) * stride;
101       }
102
103       bool describes_register(GdbServerRegister gdb_register) const {
(gdb) p feature
$1 = 2 '\002'
(gdb) p layout.feature_layouts.size()
$2 = 0       
(gdb) p reg
$3 = rr::DREG_K2

reg actually is DREG_64_YMM0H here, debugger show it as rr::DREG_K2 because both values are 60 in enum.

So there are somewhere wrong if cpu support xsave but not any avx?

[lwintermelon]

I actually have an rr recording of the rr replay crash -- captured with rr record -o rr.rr rr replay -M -a t.rr/

I don’t have time to dig into it right now, I’ll close this PR.
If you’d like to investigate, check out the attached rr trace.
rr.rr.tar.xz.zip

[khuey]

Ah, so it's not coming from gdb. Ok.

[khuey]

#4050 is a better fix.