Check for forwarding address in READ_BARRIER_MARK_REG
When the object is in the from-space, the mark bit is not set.
In this case, we can also check the lock word for being a forwarding
address. The forwarding address case happens around 25% of the time.
This CL adds the case for forwarding address lock words to
READ_BARRIER_MARK_REG.
Reduces total read barriers reaching runtime on ritzperf:
Slow paths: 20758783 -> 15457783
Deleted the mark bit check in MarkFromReadBarrier since most of the
callers check the bit now.
Perf:
ReadBarrier::Mark: 2.59% -> 2.12%
art_quick_read_barrier_mark_reg01: 0.79% -> 0.78%
art_quick_read_barrier_mark_reg00: 0.54% -> 0.50%
art_quick_read_barrier_mark_reg02: 0.31% -> 0.25%
Only X86_64 for now, will do other archs after.
Bug: 30162165
Test: test-art-host
Change-Id: Ie7289d684d0e37a887943d77710092e380457860
diff --git a/runtime/generated/asm_support_gen.h b/runtime/generated/asm_support_gen.h
index 6c189b0..f938c9f 100644
--- a/runtime/generated/asm_support_gen.h
+++ b/runtime/generated/asm_support_gen.h
@@ -98,6 +98,10 @@
DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_READ_BARRIER_STATE_MASK_TOGGLED), (static_cast<uint32_t>(art::LockWord::kReadBarrierStateMaskShiftedToggled)))
#define LOCK_WORD_THIN_LOCK_COUNT_ONE 65536
DEFINE_CHECK_EQ(static_cast<int32_t>(LOCK_WORD_THIN_LOCK_COUNT_ONE), (static_cast<int32_t>(art::LockWord::kThinLockCountOne)))
+#define LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW 0x40000000
+DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_STATE_FORWARDING_ADDRESS_OVERFLOW), (static_cast<uint32_t>(art::LockWord::kStateForwardingAddressOverflow)))
+#define LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT 0x3
+DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_STATE_FORWARDING_ADDRESS_SHIFT), (static_cast<uint32_t>(art::LockWord::kForwardingAddressShift)))
#define LOCK_WORD_GC_STATE_MASK_SHIFTED 0x30000000
DEFINE_CHECK_EQ(static_cast<uint32_t>(LOCK_WORD_GC_STATE_MASK_SHIFTED), (static_cast<uint32_t>(art::LockWord::kGCStateMaskShifted)))
#define LOCK_WORD_GC_STATE_MASK_SHIFTED_TOGGLED 0xcfffffff