Clear (madvise) card table for CC
Since the cards are not really used, we can madvise them to reduce
RAM. Image and zygote cards are cleared in pause to prevent races
between with mutators.
Pause time goes up by only 1.5us on N6P maps:
(Paused)ClearCards: Sum: 755us 99% C.I. 0.250us-28us Avg: 1.540us Max: 28us
AOSP N6P before (60s after boot system wide CC):
4,194K: .GC
4,172K: .GC
4,110K: .GC
After:
3,253K: .GC
3,205K: .GC
3,245K: .GC
Bug: 12687968
Test: test-art-host
Change-Id: I3194b5b8044c2dca427302c32d9974920fecb289
diff --git a/runtime/gc/accounting/card_table-inl.h b/runtime/gc/accounting/card_table-inl.h
index f72f219..6ff5359 100644
--- a/runtime/gc/accounting/card_table-inl.h
+++ b/runtime/gc/accounting/card_table-inl.h
@@ -50,13 +50,17 @@
}
template <bool kClearCard, typename Visitor>
-inline size_t CardTable::Scan(ContinuousSpaceBitmap* bitmap, uint8_t* scan_begin, uint8_t* scan_end,
- const Visitor& visitor, const uint8_t minimum_age) const {
+inline size_t CardTable::Scan(ContinuousSpaceBitmap* bitmap,
+ uint8_t* const scan_begin,
+ uint8_t* const scan_end,
+ const Visitor& visitor,
+ const uint8_t minimum_age) {
DCHECK_GE(scan_begin, reinterpret_cast<uint8_t*>(bitmap->HeapBegin()));
// scan_end is the byte after the last byte we scan.
DCHECK_LE(scan_end, reinterpret_cast<uint8_t*>(bitmap->HeapLimit()));
- uint8_t* card_cur = CardFromAddr(scan_begin);
- uint8_t* card_end = CardFromAddr(AlignUp(scan_end, kCardSize));
+ uint8_t* const card_begin = CardFromAddr(scan_begin);
+ uint8_t* const card_end = CardFromAddr(AlignUp(scan_end, kCardSize));
+ uint8_t* card_cur = card_begin;
CheckCardValid(card_cur);
CheckCardValid(card_end);
size_t cards_scanned = 0;
@@ -67,9 +71,6 @@
uintptr_t start = reinterpret_cast<uintptr_t>(AddrFromCard(card_cur));
bitmap->VisitMarkedRange(start, start + kCardSize, visitor);
++cards_scanned;
- if (kClearCard) {
- *card_cur = 0;
- }
}
++card_cur;
}
@@ -99,9 +100,6 @@
<< "card " << static_cast<size_t>(*card) << " intptr_t " << (start_word & 0xFF);
bitmap->VisitMarkedRange(start, start + kCardSize, visitor);
++cards_scanned;
- if (kClearCard) {
- *card = 0;
- }
}
start_word >>= 8;
start += kCardSize;
@@ -116,13 +114,14 @@
uintptr_t start = reinterpret_cast<uintptr_t>(AddrFromCard(card_cur));
bitmap->VisitMarkedRange(start, start + kCardSize, visitor);
++cards_scanned;
- if (kClearCard) {
- *card_cur = 0;
- }
}
++card_cur;
}
+ if (kClearCard) {
+ ClearCardRange(scan_begin, scan_end);
+ }
+
return cards_scanned;
}
@@ -135,7 +134,9 @@
* us to know which cards got cleared.
*/
template <typename Visitor, typename ModifiedVisitor>
-inline void CardTable::ModifyCardsAtomic(uint8_t* scan_begin, uint8_t* scan_end, const Visitor& visitor,
+inline void CardTable::ModifyCardsAtomic(uint8_t* scan_begin,
+ uint8_t* scan_end,
+ const Visitor& visitor,
const ModifiedVisitor& modified) {
uint8_t* card_cur = CardFromAddr(scan_begin);
uint8_t* card_end = CardFromAddr(AlignUp(scan_end, kCardSize));
diff --git a/runtime/gc/accounting/card_table.cc b/runtime/gc/accounting/card_table.cc
index 121da37..4506597 100644
--- a/runtime/gc/accounting/card_table.cc
+++ b/runtime/gc/accounting/card_table.cc
@@ -97,36 +97,18 @@
// Destroys MemMap via std::unique_ptr<>.
}
-void CardTable::ClearSpaceCards(space::ContinuousSpace* space) {
- // TODO: clear just the range of the table that has been modified
- uint8_t* card_start = CardFromAddr(space->Begin());
- uint8_t* card_end = CardFromAddr(space->End()); // Make sure to round up.
- memset(reinterpret_cast<void*>(card_start), kCardClean, card_end - card_start);
-}
-
void CardTable::ClearCardTable() {
static_assert(kCardClean == 0, "kCardClean must be 0");
mem_map_->MadviseDontNeedAndZero();
}
void CardTable::ClearCardRange(uint8_t* start, uint8_t* end) {
- if (!kMadviseZeroes) {
- memset(start, 0, end - start);
- return;
- }
CHECK_ALIGNED(reinterpret_cast<uintptr_t>(start), kCardSize);
CHECK_ALIGNED(reinterpret_cast<uintptr_t>(end), kCardSize);
static_assert(kCardClean == 0, "kCardClean must be 0");
uint8_t* start_card = CardFromAddr(start);
uint8_t* end_card = CardFromAddr(end);
- uint8_t* round_start = AlignUp(start_card, kPageSize);
- uint8_t* round_end = AlignDown(end_card, kPageSize);
- if (round_start < round_end) {
- madvise(round_start, round_end - round_start, MADV_DONTNEED);
- }
- // Handle unaligned regions at start / end.
- memset(start_card, 0, std::min(round_start, end_card) - start_card);
- memset(std::max(round_end, start_card), 0, end_card - std::max(round_end, start_card));
+ ZeroAndReleasePages(start_card, end_card - start_card);
}
bool CardTable::AddrIsInCardTable(const void* addr) const {
diff --git a/runtime/gc/accounting/card_table.h b/runtime/gc/accounting/card_table.h
index 969bfb7..68ef15d 100644
--- a/runtime/gc/accounting/card_table.h
+++ b/runtime/gc/accounting/card_table.h
@@ -98,15 +98,19 @@
* us to know which cards got cleared.
*/
template <typename Visitor, typename ModifiedVisitor>
- void ModifyCardsAtomic(uint8_t* scan_begin, uint8_t* scan_end, const Visitor& visitor,
+ void ModifyCardsAtomic(uint8_t* scan_begin,
+ uint8_t* scan_end,
+ const Visitor& visitor,
const ModifiedVisitor& modified);
// For every dirty at least minumum age between begin and end invoke the visitor with the
// specified argument. Returns how many cards the visitor was run on.
template <bool kClearCard, typename Visitor>
- size_t Scan(SpaceBitmap<kObjectAlignment>* bitmap, uint8_t* scan_begin, uint8_t* scan_end,
+ size_t Scan(SpaceBitmap<kObjectAlignment>* bitmap,
+ uint8_t* scan_begin,
+ uint8_t* scan_end,
const Visitor& visitor,
- const uint8_t minimum_age = kCardDirty) const
+ const uint8_t minimum_age = kCardDirty)
REQUIRES(Locks::heap_bitmap_lock_)
REQUIRES_SHARED(Locks::mutator_lock_);
@@ -119,9 +123,6 @@
// Clear a range of cards that covers start to end, start and end must be aligned to kCardSize.
void ClearCardRange(uint8_t* start, uint8_t* end);
- // Resets all of the bytes in the card table which do not map to the image space.
- void ClearSpaceCards(space::ContinuousSpace* space);
-
// Returns the first address in the heap which maps to this card.
void* AddrFromCard(const uint8_t *card_addr) const ALWAYS_INLINE;
diff --git a/runtime/gc/accounting/mod_union_table.cc b/runtime/gc/accounting/mod_union_table.cc
index 14f5997..0325535 100644
--- a/runtime/gc/accounting/mod_union_table.cc
+++ b/runtime/gc/accounting/mod_union_table.cc
@@ -168,7 +168,7 @@
bool* const contains_reference_to_other_space_;
};
-void ModUnionTableReferenceCache::ClearCards() {
+void ModUnionTableReferenceCache::ProcessCards() {
CardTable* card_table = GetHeap()->GetCardTable();
ModUnionAddToCardSetVisitor visitor(&cleared_cards_);
// Clear dirty cards in the this space and update the corresponding mod-union bits.
@@ -525,7 +525,7 @@
ModUnionTable::CardBitmap* const card_bitmap_;
};
-void ModUnionTableCardCache::ClearCards() {
+void ModUnionTableCardCache::ProcessCards() {
CardTable* const card_table = GetHeap()->GetCardTable();
ModUnionAddToCardBitmapVisitor visitor(card_bitmap_.get(), card_table);
// Clear dirty cards in the this space and update the corresponding mod-union bits.
diff --git a/runtime/gc/accounting/mod_union_table.h b/runtime/gc/accounting/mod_union_table.h
index b6792c4..591365f 100644
--- a/runtime/gc/accounting/mod_union_table.h
+++ b/runtime/gc/accounting/mod_union_table.h
@@ -55,10 +55,10 @@
virtual ~ModUnionTable() {}
- // Clear cards which map to a memory range of a space. This doesn't immediately update the
- // mod-union table, as updating the mod-union table may have an associated cost, such as
- // determining references to track.
- virtual void ClearCards() = 0;
+ // Process cards for a memory range of a space. This doesn't immediately update the mod-union
+ // table, as updating the mod-union table may have an associated cost, such as determining
+ // references to track.
+ virtual void ProcessCards() = 0;
// Set all the cards.
virtual void SetCards() = 0;
@@ -66,9 +66,9 @@
// Clear all of the table.
virtual void ClearTable() = 0;
- // Update the mod-union table using data stored by ClearCards. There may be multiple ClearCards
- // before a call to update, for example, back-to-back sticky GCs. Also mark references to other
- // spaces which are stored in the mod-union table.
+ // Update the mod-union table using data stored by ProcessCards. There may be multiple
+ // ProcessCards before a call to update, for example, back-to-back sticky GCs. Also mark
+ // references to other spaces which are stored in the mod-union table.
virtual void UpdateAndMarkReferences(MarkObjectVisitor* visitor) = 0;
// Visit all of the objects that may contain references to other spaces.
@@ -117,7 +117,7 @@
virtual ~ModUnionTableReferenceCache() {}
// Clear and store cards for a space.
- void ClearCards() OVERRIDE;
+ void ProcessCards() OVERRIDE;
// Update table based on cleared cards and mark all references to the other spaces.
void UpdateAndMarkReferences(MarkObjectVisitor* visitor) OVERRIDE
@@ -164,7 +164,7 @@
virtual ~ModUnionTableCardCache() {}
// Clear and store cards for a space.
- virtual void ClearCards() OVERRIDE;
+ virtual void ProcessCards() OVERRIDE;
// Mark all references to the alloc space(s).
virtual void UpdateAndMarkReferences(MarkObjectVisitor* visitor) OVERRIDE
diff --git a/runtime/gc/accounting/mod_union_table_test.cc b/runtime/gc/accounting/mod_union_table_test.cc
index 2810f58..cf63b30 100644
--- a/runtime/gc/accounting/mod_union_table_test.cc
+++ b/runtime/gc/accounting/mod_union_table_test.cc
@@ -214,7 +214,7 @@
ASSERT_TRUE(other_space_ref2 != nullptr);
obj1->Set(1, other_space_ref1);
obj2->Set(3, other_space_ref2);
- table->ClearCards();
+ table->ProcessCards();
std::set<mirror::Object*> visited_before;
CollectVisitedVisitor collector_before(&visited_before);
table->UpdateAndMarkReferences(&collector_before);
diff --git a/runtime/gc/accounting/space_bitmap.cc b/runtime/gc/accounting/space_bitmap.cc
index e2f5a1d..f4d0bc7 100644
--- a/runtime/gc/accounting/space_bitmap.cc
+++ b/runtime/gc/accounting/space_bitmap.cc
@@ -118,31 +118,8 @@
}
const uintptr_t start_index = OffsetToIndex(begin_offset);
const uintptr_t end_index = OffsetToIndex(end_offset);
- Atomic<uintptr_t>* const mem_begin = &bitmap_begin_[start_index];
- Atomic<uintptr_t>* const mem_end = &bitmap_begin_[end_index];
- Atomic<uintptr_t>* const page_begin = AlignUp(mem_begin, kPageSize);
- Atomic<uintptr_t>* const page_end = AlignDown(mem_end, kPageSize);
- if (!kMadviseZeroes || page_begin >= page_end) {
- // No possible area to madvise.
- std::fill(reinterpret_cast<uint8_t*>(mem_begin),
- reinterpret_cast<uint8_t*>(mem_end),
- 0);
- } else {
- // Spans one or more pages.
- DCHECK_LE(mem_begin, page_begin);
- DCHECK_LE(page_begin, page_end);
- DCHECK_LE(page_end, mem_end);
- std::fill(reinterpret_cast<uint8_t*>(mem_begin),
- reinterpret_cast<uint8_t*>(page_begin),
- 0);
- CHECK_NE(madvise(page_begin,
- reinterpret_cast<uint8_t*>(page_end) - reinterpret_cast<uint8_t*>(page_begin),
- MADV_DONTNEED),
- -1) << "madvise failed";
- std::fill(reinterpret_cast<uint8_t*>(page_end),
- reinterpret_cast<uint8_t*>(mem_end),
- 0);
- }
+ ZeroAndReleasePages(reinterpret_cast<uint8_t*>(&bitmap_begin_[start_index]),
+ (end_index - start_index) * sizeof(*bitmap_begin_));
}
template<size_t kAlignment>
diff --git a/runtime/gc/collector/concurrent_copying.cc b/runtime/gc/collector/concurrent_copying.cc
index 13af67e..c43b048 100644
--- a/runtime/gc/collector/concurrent_copying.cc
+++ b/runtime/gc/collector/concurrent_copying.cc
@@ -486,9 +486,14 @@
// Table is non null for boot image and zygote spaces. It is only null for application image
// spaces.
if (table != nullptr) {
- // TODO: Add preclean outside the pause.
- table->ClearCards();
+ // TODO: Consider adding precleaning outside the pause.
+ table->ProcessCards();
table->VisitObjects(GrayImmuneObjectVisitor::Callback, &visitor);
+ // Since the cards are recorded in the mod-union table and this is paused, we can clear
+ // the cards for the space (to madvise).
+ TimingLogger::ScopedTiming split2("(Paused)ClearCards", GetTimings());
+ card_table->ClearCardRange(space->Begin(),
+ AlignDown(space->End(), accounting::CardTable::kCardSize));
} else {
// TODO: Consider having a mark bitmap for app image spaces and avoid scanning during the
// pause because app image spaces are all dirty pages anyways.
@@ -2325,9 +2330,14 @@
MutexLock mu(self, mark_stack_lock_);
CHECK_EQ(pooled_mark_stacks_.size(), kMarkStackPoolSize);
}
- region_space_ = nullptr;
{
- MutexLock mu(Thread::Current(), skipped_blocks_lock_);
+ TimingLogger::ScopedTiming split("ClearRegionSpaceCards", GetTimings());
+ // We do not currently use the region space cards at all, madvise them away to save ram.
+ heap_->GetCardTable()->ClearCardRange(region_space_->Begin(), region_space_->Limit());
+ region_space_ = nullptr;
+ }
+ {
+ MutexLock mu(self, skipped_blocks_lock_);
skipped_blocks_map_.clear();
}
{
@@ -2339,10 +2349,9 @@
if (kUseBakerReadBarrier && kFilterModUnionCards) {
TimingLogger::ScopedTiming split("FilterModUnionCards", GetTimings());
ReaderMutexLock mu2(self, *Locks::heap_bitmap_lock_);
- gc::Heap* const heap = Runtime::Current()->GetHeap();
for (space::ContinuousSpace* space : immune_spaces_.GetSpaces()) {
DCHECK(space->IsImageSpace() || space->IsZygoteSpace());
- accounting::ModUnionTable* table = heap->FindModUnionTableFromSpace(space);
+ accounting::ModUnionTable* table = heap_->FindModUnionTableFromSpace(space);
// Filter out cards that don't need to be set.
if (table != nullptr) {
table->FilterCards();
@@ -2351,7 +2360,7 @@
}
if (kUseBakerReadBarrier) {
TimingLogger::ScopedTiming split("EmptyRBMarkBitStack", GetTimings());
- DCHECK(rb_mark_bit_stack_.get() != nullptr);
+ DCHECK(rb_mark_bit_stack_ != nullptr);
const auto* limit = rb_mark_bit_stack_->End();
for (StackReference<mirror::Object>* it = rb_mark_bit_stack_->Begin(); it != limit; ++it) {
CHECK(it->AsMirrorPtr()->AtomicSetMarkBit(1, 0));
diff --git a/runtime/gc/heap.cc b/runtime/gc/heap.cc
index 3b9abd2..ffad80d 100644
--- a/runtime/gc/heap.cc
+++ b/runtime/gc/heap.cc
@@ -3327,7 +3327,7 @@
const char* name = space->IsZygoteSpace() ? "ZygoteModUnionClearCards" :
"ImageModUnionClearCards";
TimingLogger::ScopedTiming t2(name, timings);
- table->ClearCards();
+ table->ProcessCards();
} else if (use_rem_sets && rem_set != nullptr) {
DCHECK(collector::SemiSpace::kUseRememberedSet && collector_type_ == kCollectorTypeGSS)
<< static_cast<int>(collector_type_);
diff --git a/runtime/mem_map.cc b/runtime/mem_map.cc
index bb07fcb..e4c67e0 100644
--- a/runtime/mem_map.cc
+++ b/runtime/mem_map.cc
@@ -918,4 +918,23 @@
}
}
+void ZeroAndReleasePages(void* address, size_t length) {
+ uint8_t* const mem_begin = reinterpret_cast<uint8_t*>(address);
+ uint8_t* const mem_end = mem_begin + length;
+ uint8_t* const page_begin = AlignUp(mem_begin, kPageSize);
+ uint8_t* const page_end = AlignDown(mem_end, kPageSize);
+ if (!kMadviseZeroes || page_begin >= page_end) {
+ // No possible area to madvise.
+ std::fill(mem_begin, mem_end, 0);
+ } else {
+ // Spans one or more pages.
+ DCHECK_LE(mem_begin, page_begin);
+ DCHECK_LE(page_begin, page_end);
+ DCHECK_LE(page_end, mem_end);
+ std::fill(mem_begin, page_begin, 0);
+ CHECK_NE(madvise(page_begin, page_end - page_begin, MADV_DONTNEED), -1) << "madvise failed";
+ std::fill(page_end, mem_end, 0);
+ }
+}
+
} // namespace art
diff --git a/runtime/mem_map.h b/runtime/mem_map.h
index 597f0d4..049ae12 100644
--- a/runtime/mem_map.h
+++ b/runtime/mem_map.h
@@ -241,9 +241,13 @@
friend class MemMapTest; // To allow access to base_begin_ and base_size_.
};
+
std::ostream& operator<<(std::ostream& os, const MemMap& mem_map);
std::ostream& operator<<(std::ostream& os, const MemMap::Maps& mem_maps);
+// Zero and release pages if possible, no requirements on alignments.
+void ZeroAndReleasePages(void* address, size_t length);
+
} // namespace art
#endif // ART_RUNTIME_MEM_MAP_H_